From 3a38d4ed50c62f5fcf6a083f307b99f27e2e07ad Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 14:01:05 +0300 Subject: [PATCH 01/41] Doc fix: delete VerticalRaw format and remove escaping from Pretty and Vertical formats --- docs/en/interfaces/formats.md | 44 ++++++++++++++--------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index c47e75142df..54d839a99d3 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -14,7 +14,6 @@ The table below lists supported formats and how they can be used in `INSERT` and | [CSVWithNames](#csvwithnames) | ✔ | ✔ | | [Values](#values) | ✔ | ✔ | | [Vertical](#vertical) | ✗ | ✔ | -| [VerticalRaw](#verticalraw) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | @@ -355,6 +354,19 @@ SELECT * FROM t_null └───┴──────┘ ``` +Rows are not escaped in `Pretty` format: + +``` sql +:) SELECT 'String with \'quotes\' and \t character' AS Test_escaping +FORMAT Pretty + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Test_escaping ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ String with 'quotes' and character │ +└───────────────────────────────────────────┘ +``` + To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). @@ -461,38 +473,18 @@ Row 1: x: 1 y: ᴺᵁᴸᴸ ``` - -This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). - -## VerticalRaw {#verticalraw} - -Differs from `Vertical` format in that the rows are not escaped. -This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). - -Examples: - -``` -:) SHOW CREATE TABLE geonames FORMAT VerticalRaw; -Row 1: -────── -statement: CREATE TABLE default.geonames ( geonameid UInt32, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192) - -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw; -Row 1: -────── -test: string with 'quotes' and with some special - characters -``` - -Compare with the Vertical format: +Rows are not escaped in `Vertical` format: ``` :) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical; Row 1: ────── -test: string with \'quotes\' and \t with some special \n characters +test: string with 'quotes' and with some special + characters ``` +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). + ## XML {#xml} XML format is suitable only for output, not for parsing. Example: From fd7c1589bb573475a3d007ebd73227f5359eff26 Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 14:10:55 +0300 Subject: [PATCH 02/41] Doc fix: editing text about escaping --- docs/en/interfaces/formats.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 54d839a99d3..cdb1422b480 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -357,11 +357,11 @@ SELECT * FROM t_null Rows are not escaped in `Pretty` format: ``` sql -:) SELECT 'String with \'quotes\' and \t character' AS Test_escaping -FORMAT Pretty +:) SELECT 'String with \'quotes\' and \t character' AS Escaping_test +FORMAT Pretty; ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Test_escaping ┃ +┃ Escaping_test ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ │ String with 'quotes' and character │ └───────────────────────────────────────────┘ From a650e1ed08095bccbb3608243d20d7d7e86e662f Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 15:07:34 +0300 Subject: [PATCH 03/41] Doc fix: fix example in pretty format --- docs/en/interfaces/formats.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index cdb1422b480..9c888eaa3ad 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -345,11 +345,13 @@ Each result block is output as a separate table. This is necessary so that block [NULL](../query_language/syntax.md) is output as `ᴺᵁᴸᴸ`. ``` sql -SELECT * FROM t_null +SELECT * FROM t_null FORMAT Pretty; ``` ``` -┌─x─┬────y─┐ +┏━━━┳━━━━━━┓ +┃ x ┃ y ┃ +┡━━━╇━━━━━━┩ │ 1 │ ᴺᵁᴸᴸ │ └───┴──────┘ ``` @@ -359,12 +361,14 @@ Rows are not escaped in `Pretty` format: ``` sql :) SELECT 'String with \'quotes\' and \t character' AS Escaping_test FORMAT Pretty; +``` -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Escaping_test ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ String with 'quotes' and character │ -└───────────────────────────────────────────┘ +``` +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Escaping_test ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ String with 'quotes' and character │ +└──────────────────────────────────────────┘ ``` To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. From 996263490c9314e68e7ce65cfff8367a392eec73 Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 15:59:25 +0300 Subject: [PATCH 04/41] Doc fix: fix example in pretty format --- docs/en/interfaces/formats.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 9c888eaa3ad..1d32406af34 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -344,37 +344,34 @@ Each result block is output as a separate table. This is necessary so that block [NULL](../query_language/syntax.md) is output as `ᴺᵁᴸᴸ`. +Example (shown for the [PrettyCompact](#prettycompact) format): + ``` sql -SELECT * FROM t_null FORMAT Pretty; +SELECT * FROM t_null FORMAT ``` -``` -┏━━━┳━━━━━━┓ -┃ x ┃ y ┃ -┡━━━╇━━━━━━┩ +``` text +┌─x─┬─y────┐ │ 1 │ ᴺᵁᴸᴸ │ └───┴──────┘ ``` -Rows are not escaped in `Pretty` format: +Rows are not escaped in Pretty* format. Example is shown for the [PrettyCompact](#prettycompact) format: ``` sql -:) SELECT 'String with \'quotes\' and \t character' AS Escaping_test -FORMAT Pretty; +SELECT 'String with \'quotes\' and \t character' AS Escaping_test ``` -``` -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Escaping_test ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ String with 'quotes' and character │ +``` +┌─Escaping_test────────────────────────────┐ +│ String with 'quotes' and character │ └──────────────────────────────────────────┘ ``` To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). -The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the PrettyCompact format): +The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the [PrettyCompact](#prettycompact) format): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact @@ -477,10 +474,13 @@ Row 1: x: 1 y: ᴺᵁᴸᴸ ``` -Rows are not escaped in `Vertical` format: +Rows are not escaped in Vertical format: + +``` sql +SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical +``` ``` -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical; Row 1: ────── test: string with 'quotes' and with some special From 0ce656856a01a2e0977b1660de57fcbc55adc409 Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 16:05:40 +0300 Subject: [PATCH 05/41] Doc fix: last fixes --- docs/en/interfaces/formats.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 1d32406af34..24522507b71 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -347,10 +347,10 @@ Each result block is output as a separate table. This is necessary so that block Example (shown for the [PrettyCompact](#prettycompact) format): ``` sql -SELECT * FROM t_null FORMAT +SELECT * FROM t_null ``` -``` text +``` ┌─x─┬─y────┐ │ 1 │ ᴺᵁᴸᴸ │ └───┴──────┘ From 64eb46fa6bf910575cbac40c0bf64273bc19ae4e Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 16:14:43 +0300 Subject: [PATCH 06/41] Doc fix: last fixes --- docs/en/interfaces/formats.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 24522507b71..5e0c9801ff0 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -347,11 +347,11 @@ Each result block is output as a separate table. This is necessary so that block Example (shown for the [PrettyCompact](#prettycompact) format): ``` sql -SELECT * FROM t_null +SELECT * FROM t_null ``` ``` -┌─x─┬─y────┐ +┌─x─┬────y─┐ │ 1 │ ᴺᵁᴸᴸ │ └───┴──────┘ ``` @@ -363,9 +363,9 @@ SELECT 'String with \'quotes\' and \t character' AS Escaping_test ``` ``` -┌─Escaping_test────────────────────────────┐ -│ String with 'quotes' and character │ -└──────────────────────────────────────────┘ +┌─Escaping_test────────────────────────┐ +│ String with 'quotes' and character │ +└──────────────────────────────────────┘ ``` To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. @@ -402,7 +402,7 @@ Extremes: ## PrettyCompact {#prettycompact} -Differs from `Pretty` in that the grid is drawn between rows and the result is more compact. +Differs from [Pretty](#pretty) in that the grid is drawn between rows and the result is more compact. This format is used by default in the command-line client in interactive mode. ## PrettyCompactMonoBlock {#prettycompactmonoblock} @@ -483,7 +483,7 @@ SELECT 'string with \'quotes\' and \t with some special \n characters' AS test F ``` Row 1: ────── -test: string with 'quotes' and with some special +test: string with 'quotes' and with some special characters ``` From 632398640646111d9753ecdff4d2d256da31c28e Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 16:17:24 +0300 Subject: [PATCH 07/41] Doc fix --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 5e0c9801ff0..eddefaa9394 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -356,7 +356,7 @@ SELECT * FROM t_null └───┴──────┘ ``` -Rows are not escaped in Pretty* format. Example is shown for the [PrettyCompact](#prettycompact) format: +Rows are not escaped in Pretty* formats. Example is shown for the [PrettyCompact](#prettycompact) format: ``` sql SELECT 'String with \'quotes\' and \t character' AS Escaping_test From 3dac4f018b22614093170f644235e9bc52062c59 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Feb 2019 12:06:10 +0300 Subject: [PATCH 08/41] DOCAPI-5441: The tcp_port_secure setting description is added. --- docs/en/operations/server_settings/settings.md | 14 ++++++++++++-- docs/en/operations/troubleshooting.md | 4 ++-- .../dicts/external_dicts_dict_sources.md | 2 +- docs/en/query_language/table_functions/remote.md | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index f339fb6ce28..be1b35f038b 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -196,7 +196,7 @@ For more details, see [GraphiteMergeTree](../../operations/table_engines/graphit The port for connecting to the server over HTTP(s). -If `https_port` is specified, [openSSL](#openssl) must be configured. +If `https_port` is specified, [openSSL](#server_settings-openssl) must be configured. If `http_port` is specified, the openSSL configuration is ignored even if it is set. @@ -430,7 +430,7 @@ For more information, see the MergeTreeSettings.h header file. ``` -## openSSL +## openSSL {#server_settings-openssl} SSL client/server configuration. @@ -609,6 +609,16 @@ Port for communicating with clients over the TCP protocol. 9000 ``` +## tcp_port_secure {#server_settings-tcp_port_secure} + +Port for communicating with clients over the secure connection by TCP protocol. Use it with [OpenSSL](#server_settings-openssl) settings. + +**Example** + +```xml +9440 +``` + ## tmp_path diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 445cd67bb7c..fdfff6b26b2 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -111,8 +111,8 @@ Check: Check: - - The `tcp_port_secure` setting. - - Settings for SSL sertificates. + - The [tcp_port_secure](server_settings/settings.md#server_settings-tcp_port_secure) setting. + - Settings for [SSL sertificates](server_settings/settings.md#server_settings-openssl). Use proper parameters while connecting. For example, use the `port_secure` parameter with `clickhouse_client`. diff --git a/docs/en/query_language/dicts/external_dicts_dict_sources.md b/docs/en/query_language/dicts/external_dicts_dict_sources.md index f26967c2d0f..8fb2145ecaf 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_sources.md +++ b/docs/en/query_language/dicts/external_dicts_dict_sources.md @@ -88,7 +88,7 @@ Example of settings: ``` -In order for ClickHouse to access an HTTPS resource, you must [configure openSSL](../../operations/server_settings/settings.md) in the server configuration. +In order for ClickHouse to access an HTTPS resource, you must [configure openSSL](../../operations/server_settings/settings.md#server_settings-openssl) in the server configuration. Setting fields: diff --git a/docs/en/query_language/table_functions/remote.md b/docs/en/query_language/table_functions/remote.md index 06d9876f1c7..3b9263d959c 100644 --- a/docs/en/query_language/table_functions/remote.md +++ b/docs/en/query_language/table_functions/remote.md @@ -72,6 +72,6 @@ The `remote` table function can be useful in the following cases: If the user is not specified, `default` is used. If the password is not specified, an empty password is used. -`remoteSecure` - same as `remote` but with secured connection. Default port - `tcp_port_secure` from config or 9440. +`remoteSecure` - same as `remote` but with secured connection. Default port — [tcp_port_secure](../../operations/server_settings/settings.md#server_settings-tcp_port_secure) from config or 9440. [Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/remote/) From 455ca955e569b5e6a23605c828a90686f1ca4567 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Feb 2019 14:20:27 +0300 Subject: [PATCH 09/41] DOCAPI-5203: Added the descriptions for direct I/O settings for MergeTree merge and SELECT queries. --- dbms/src/Interpreters/Settings.h | 2 +- .../en/operations/server_settings/settings.md | 2 +- docs/en/operations/settings/settings.md | 28 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 42e60a2353e..0b0f630b49f 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -103,7 +103,7 @@ struct Settings \ M(SettingUInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ") \ \ - M(SettingUInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for input/output operations is bypassing the page cache. 0 - disabled.") \ + M(SettingUInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for reading the data with O_DIRECT option during SELECT queries execution. 0 - disabled.") \ \ M(SettingBool, force_index_by_date, 0, "Throw an exception if there is a partition key in a table, and it is not used.") \ M(SettingBool, force_primary_key, 0, "Throw an exception if there is primary key in a table, and it is not used.") \ diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index f339fb6ce28..db09323bab9 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -417,7 +417,7 @@ The value 0 means that you can delete all tables without any restrictions. ## merge_tree {#server_settings-merge_tree} -Fine tuning for tables in the [ MergeTree](../../operations/table_engines/mergetree.md). +Fine tuning for tables in the [MergeTree](../../operations/table_engines/mergetree.md). For more information, see the MergeTreeSettings.h header file. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e30758fc943..2fa8f37e404 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -175,6 +175,34 @@ Any positive integer. **Default value**: 1048576. +## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} + +The threshold for using direct I/O interface of the storage disk. + +ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. + +**Possible values** + +Positive integer. + +0 — The direct I/O is disabled. + +**Default value**: `10 * 1024 * 1024 * 1024` bytes. + +## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} + +The threshold for using direct I/O interface of the storage disk. + +ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, than ClickHouse reads the data from the storage disk directly with `O_DIRECT` option. + +**Possible values** + +Positive integer. + +0 — The direct I/O is disabled. + +**Default value**: 0. + ## log_queries Setting up query logging. From 0613cd26bcec37389bc8bd0b871f269de332fa48 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Feb 2019 19:53:11 +0300 Subject: [PATCH 10/41] DOCAPI-5441. Some restructurization for tcp_port_secure. --- docs/en/operations/server_settings/settings.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index be1b35f038b..952aae6979a 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -613,13 +613,16 @@ Port for communicating with clients over the TCP protocol. Port for communicating with clients over the secure connection by TCP protocol. Use it with [OpenSSL](#server_settings-openssl) settings. -**Example** +**Possible values** + +Positive integer. + +**Default value** ```xml -9440 +9440 ``` - ## tmp_path Path to temporary data for processing large queries. From 47c82b6c08ddfc5af6ec2e64134928a7c9d6ccfc Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 17:01:07 +0300 Subject: [PATCH 11/41] DOCAPI-5441: tcp_port_secure topic fix. --- docs/en/operations/server_settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 952aae6979a..d21b0379a90 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -611,7 +611,7 @@ Port for communicating with clients over the TCP protocol. ## tcp_port_secure {#server_settings-tcp_port_secure} -Port for communicating with clients over the secure connection by TCP protocol. Use it with [OpenSSL](#server_settings-openssl) settings. +Port for communicating with the clients over the secure connection by TCP protocol. Use it with [OpenSSL](#server_settings-openssl) settings. **Possible values** From 646ba6726f3b5c48fe1065a61f8e31d21342b836 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 17:37:56 +0300 Subject: [PATCH 12/41] DOCAPI-5203: Clarification of direct I/O settings descriptions. --- docs/en/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2fa8f37e404..ff31b267d1d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -177,7 +177,7 @@ Any positive integer. ## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} -The threshold for using direct I/O interface of the storage disk. +The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. @@ -191,9 +191,9 @@ Positive integer. ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} -The threshold for using direct I/O interface of the storage disk. +The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. -ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, than ClickHouse reads the data from the storage disk directly with `O_DIRECT` option. +ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, then ClickHouse reads the data from the storage disk with `O_DIRECT` option. **Possible values** From fe6c988c5543a6813a3403b44078d0b4da3772fd Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Mon, 4 Mar 2019 17:52:12 +0300 Subject: [PATCH 13/41] Doc fixes --- docs/en/operations/backup.md | 4 +- docs/ru/interfaces/formats.md | 44 +++++++++---------- docs/ru/operations/settings/settings.md | 2 +- .../table_engines/collapsingmergetree.md | 2 +- .../operations/table_engines/replication.md | 2 +- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index fddcaba78f7..f9b0d134815 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -1,6 +1,6 @@ # Data Backup -While [replication](table_engines/replication.md#table_engines-replication) provides protection from hardware failures, it does not protect against human errors: accidentally deleting data, deleting the wrong table or on the wrong cluster, software bugs leading to incorrect data processing or data corruption. In many cases commands like these will affect all replicas. ClickHouse has built-in safeguards to prevent some of mistakes, for example by default [you can't just drop tables with MergeTree-like engine containing more than 50Gb of data](https://github.com/yandex/ClickHouse/blob/v18.14.18-stable/dbms/programs/server/config.xml), but they don't cover all possible cases and can be circumvented. +While [replication](table_engines/replication.md) provides protection from hardware failures, it does not protect against human errors: accidentally deleting data, deleting the wrong table or on the wrong cluster, software bugs leading to incorrect data processing or data corruption. In many cases commands like these will affect all replicas. ClickHouse has built-in safeguards to prevent some of mistakes, for example by default [you can't just drop tables with MergeTree-like engine containing more than 50Gb of data](https://github.com/yandex/ClickHouse/blob/v18.14.18-stable/dbms/programs/server/config.xml), but they don't cover all possible cases and can be circumvented. So in order to effectively mitigate possible human errors, you should carefully prepare your backup and restore strategy **in advance**. @@ -35,4 +35,4 @@ There's a third-party tool to automate this approach: [clickhouse-backup](https: [Console client](../interfaces/cli.md) can be used to run queries like `SELECT * FROM my_table;` to dump the tables into files using any of the [supported serialization formats](../interfaces/formats.md#formats). Though if you are using ClickHouse as intended and have large enough volumes of data, this will hardly be practical. -[Original article](https://clickhouse.yandex/docs/en/operations/access_rights/) \ No newline at end of file +[Original article](https://clickhouse.yandex/docs/en/operations/access_rights/) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 303ed85cd73..e345ef651d3 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -14,7 +14,6 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT [CSVWithNames](#csvwithnames) | ✔ | ✔ | [Values](#values) | ✔ | ✔ | [Vertical](#vertical) | ✗ | ✔ | -[VerticalRaw](#verticalraw) | ✗ | ✔ | [JSON](#json) | ✗ | ✔ | [JSONCompact](#jsoncompact) | ✗ | ✔ | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | @@ -353,10 +352,22 @@ SELECT * FROM t_null └───┴──────┘ ``` +В форматах `Pretty*` строки выводятся без экранирования. Ниже приведен пример для формата [PrettyCompact](#prettycompact): + +``` sql +SELECT 'String with \'quotes\' and \t character' AS Escaping_test +``` + +``` +┌─Escaping_test────────────────────────┐ +│ String with 'quotes' and character │ +└──────────────────────────────────────┘ +``` + Для защиты от вываливания слишком большого количества данных в терминал, выводится только первые 10 000 строк. Если строк больше или равно 10 000, то будет написано "Showed first 10 000." Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). -Формат Pretty поддерживает вывод тотальных значений (при использовании WITH TOTALS) и экстремальных значений (при настройке extremes выставленной в 1). В этих случаях, после основных данных выводятся тотальные значения, и экстремальные значения, в отдельных табличках. Пример (показан для формата PrettyCompact): +Формат `Pretty` поддерживает вывод тотальных значений (при использовании WITH TOTALS) и экстремальных значений (при настройке extremes выставленной в 1). В этих случаях, после основных данных выводятся тотальные значения, и экстремальные значения, в отдельных табличках. Пример (показан для формата [PrettyCompact](#prettycompact)): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact @@ -387,7 +398,7 @@ Extremes: ## PrettyCompact {#prettycompact} -Отличается от `Pretty` тем, что не рисуется сетка между строками - результат более компактный. +Отличается от [Pretty](#pretty) тем, что не рисуется сетка между строками - результат более компактный. Этот формат используется по умолчанию в клиенте командной строки в интерактивном режиме. ## PrettyCompactMonoBlock {#prettycompactmonoblock} @@ -432,6 +443,7 @@ FixedString представлены просто как последовате Array представлены как длина в формате varint (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), а затем элементы массива, подряд. Для поддержки [NULL](../query_language/syntax.md#null-literal) перед каждым значением типа [Nullable](../data_types/nullable.md + ## Values Выводит каждую строку в скобках. Строки разделены запятыми. После последней строки запятой нет. Значения внутри скобок также разделены запятыми. Числа выводятся в десятичном виде без кавычек. Массивы выводятся в квадратных скобках. Строки, даты, даты-с-временем выводятся в кавычках. Правила экранирования и особенности парсинга аналогичны формату [TabSeparated](#tabseparated). При форматировании, лишние пробелы не ставятся, а при парсинге - допустимы и пропускаются (за исключением пробелов внутри значений типа массив, которые недопустимы). [NULL](../query_language/syntax.md) представляется как `NULL`. @@ -458,34 +470,20 @@ x: 1 y: ᴺᵁᴸᴸ ``` -Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). +В формате `Vertical` строки выводятся без экранирования. Например: -## VerticalRaw {#verticalraw} - -Отличается от формата `Vertical` тем, что строки выводятся без экранирования. -Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). - -Примеры: +``` sql +SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical ``` -:) SHOW CREATE TABLE geonames FORMAT VerticalRaw; -Row 1: -────── -statement: CREATE TABLE default.geonames ( geonameid UInt32, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192) -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw; +``` Row 1: ────── -test: string with 'quotes' and with some special +test: string with 'quotes' and with some special characters ``` -Для сравнения - формат Vertical: -``` -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical; -Row 1: -────── -test: string with \'quotes\' and \t with some special \n characters -``` +Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). ## XML {#xml} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index ed1395a1aaf..c174507859b 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -118,7 +118,7 @@ ClickHouse применяет настройку в тех случаях, ко Это намного больше, чем `max_block_size`. Это сделано, потому что некоторые движки таблиц (`*MergeTree`) будут на каждый вставляемый блок формировать кусок данных на диске, что является довольно большой сущностью. Также, в таблицах типа `*MergeTree`, данные сортируются при вставке, и достаточно большой размер блока позволяет отсортировать больше данных в оперативке. -## max_replica_delay_for_distributed_queries {#settings_settings_max_replica_delay_for_distributed_queries} +## max_replica_delay_for_distributed_queries {#settings-max_replica_delay_for_distributed_queries} Отключает отстающие реплики при распределенных запросах. Смотрите "[Репликация](../../operations/table_engines/replication.md)". diff --git a/docs/ru/operations/table_engines/collapsingmergetree.md b/docs/ru/operations/table_engines/collapsingmergetree.md index cc938de2af3..f09d84cf8e8 100644 --- a/docs/ru/operations/table_engines/collapsingmergetree.md +++ b/docs/ru/operations/table_engines/collapsingmergetree.md @@ -54,7 +54,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -## Сворачивание (удаление) строк {table_engine-collapsingmergetree-collapsing} +## Сворачивание (удаление) строк {#table_engine-collapsingmergetree-collapsing} ### Данные diff --git a/docs/ru/operations/table_engines/replication.md b/docs/ru/operations/table_engines/replication.md index fe61e3a232c..efe5a2257b3 100644 --- a/docs/ru/operations/table_engines/replication.md +++ b/docs/ru/operations/table_engines/replication.md @@ -67,7 +67,7 @@ Система следит за синхронностью данных на репликах и умеет восстанавливаться после сбоя. Восстановление после сбоя автоматическое (в случае небольших различий в данных) или полуавтоматическое (когда данные отличаются слишком сильно, что может свидетельствовать об ошибке конфигурации). -## Создание реплицируемых таблиц +## Создание реплицируемых таблиц {#creating-replicated-tables} В начало имени движка таблицы добавляется `Replicated`. Например, `ReplicatedMergeTree`. From 4bae599fcdcac0394917c26d1d201fcd4a8c9479 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 18:34:35 +0300 Subject: [PATCH 14/41] DOCAPI-5203: The direct I/O settings are moved to right places. --- docs/en/operations/settings/settings.md | 14 -------------- docs/en/operations/table_engines/mergetree.md | 2 ++ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index ff31b267d1d..8f3e0d525ef 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -175,20 +175,6 @@ Any positive integer. **Default value**: 1048576. -## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} - -The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. - -ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. - -**Possible values** - -Positive integer. - -0 — The direct I/O is disabled. - -**Default value**: `10 * 1024 * 1024 * 1024` bytes. - ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 7d86776d27b..3658392c70c 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -70,6 +70,8 @@ For a description of request parameters, see [request description](../../query_l - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + **Example of sections setting** From 9fde4594c92a9f4d975ed299298c7c9564729e4e Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 19:27:00 +0300 Subject: [PATCH 15/41] DOCAPI-5758: New parameter kafka_skip_broken_messages in the Kafka engine description. --- docs/en/operations/table_engines/kafka.md | 63 ++++++++++++++--------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/docs/en/operations/table_engines/kafka.md b/docs/en/operations/table_engines/kafka.md index 3927e472e50..87322afb48e 100644 --- a/docs/en/operations/table_engines/kafka.md +++ b/docs/en/operations/table_engines/kafka.md @@ -9,38 +9,38 @@ Kafka lets you: - Process streams as they become available. -Old format: +## Creating a Table {#table_engine-kafka-creating-a-table} ``` -Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format - [, kafka_row_delimiter, kafka_schema, kafka_num_consumers]) +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = Kafka() +SETTINGS + kafka_broker_list = 'host:port', + kafka_topic_list = 'topic1,topic2,...', + kafka_group_name = 'group_name', + kafka_format = 'data_format'[,] + [kafka_row_delimiter = 'delimiter_symbol',] + [kafka_schema = '',] + [kafka_num_consumers = N,] + [kafka_skip_broken_messages = <0|1>] ``` - -New format: - -``` -Kafka SETTINGS - kafka_broker_list = 'localhost:9092', - kafka_topic_list = 'topic1,topic2', - kafka_group_name = 'group1', - kafka_format = 'JSONEachRow', - kafka_row_delimiter = '\n', - kafka_schema = '', - kafka_num_consumers = 2 -``` - Required parameters: -- `kafka_broker_list` – A comma-separated list of brokers (`localhost:9092`). -- `kafka_topic_list` – A list of Kafka topics (`my_topic`). -- `kafka_group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. -- `kafka_format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the "Formats" section. +- `kafka_broker_list` – A comma-separated list of brokers (for example, `localhost:9092`). +- `kafka_topic_list` – A list of Kafka topics. +- `kafka_group_name` – A group of Kafka consumers. Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `kafka_format` – Message format. Uses the same notation as the SQL `FORMAT` function, such as ` JSONEachRow`. For more information, see the [Formats](../../interfaces/formats.md) section. Optional parameters: -- `kafka_row_delimiter` - Character-delimiter of records (rows), which ends the message. -- `kafka_schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. +- `kafka_row_delimiter` – Delimiter character, which ends the message. +- `kafka_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. - `kafka_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition. +- `kafka_skip_broken_messages` – Mode of Kafka messages parser. If `kafka_skip_broken_messages = 1` then the engine skips the Kafka messages (message equals a row of data) that can't be parsed. Examples: @@ -72,6 +72,23 @@ Examples: kafka_num_consumers = 4; ``` + +
Deprecated Method for Creating a Table + +!!! attention + Do not use this method in new projects and, if possible, switch the old projects to the method described above. + + +``` +Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format + [, kafka_row_delimiter, kafka_schema, kafka_num_consumers, kafka_skip_broken_messages]) +``` + +
+ +## Description + + The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. Read more about this at [http://kafka.apache.org/intro](http://kafka.apache.org/intro). From 16c967002e528644963a8600cd8a97458c34005a Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 5 Mar 2019 13:16:45 +0300 Subject: [PATCH 16/41] DOCAPI-5203: Fix in terms of direct I/O. --- docs/en/operations/settings/settings.md | 2 +- docs/en/operations/table_engines/mergetree.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8f3e0d525ef..fff660999e2 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -177,7 +177,7 @@ Any positive integer. ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} -The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. +The minimum data volume to be read from storage required for using of the direct I/O access to the storage disk. ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, then ClickHouse reads the data from the storage disk with `O_DIRECT` option. diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 3658392c70c..53bb909b16d 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -70,7 +70,7 @@ For a description of request parameters, see [request description](../../query_l - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O access to the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. **Example of sections setting** From 52c9d4e1ad001de14f25e81e589fc9124b51c12d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Mar 2019 13:55:26 +0300 Subject: [PATCH 17/41] (De)Serialize set index with multiple bulks. #4486 --- .../src/Storages/MergeTree/MergeTreeIndices.h | 1 + .../MergeTree/MergeTreeSetSkippingIndex.cpp | 23 +++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndices.h b/dbms/src/Storages/MergeTree/MergeTreeIndices.h index 6738d667b44..1d62e9e9e9c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndices.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndices.h @@ -11,6 +11,7 @@ #include #include #include +#include constexpr auto INDEX_FILE_PREFIX = "skp_idx_"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index 69323fe8bb4..686d36b2c5b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -47,7 +47,16 @@ void MergeTreeSetIndexGranule::serializeBinary(WriteBuffer & ostr) const for (size_t i = 0; i < index.columns.size(); ++i) { const auto & type = index.data_types[i]; - type->serializeBinaryBulk(*columns[i], ostr, 0, size()); + + IDataType::SerializeBinaryBulkSettings settings; + settings.getter = [&ostr](IDataType::SubstreamPath) -> WriteBuffer * { return &ostr; }; + settings.position_independent_encoding = false; + settings.low_cardinality_max_dictionary_size = 0; + + IDataType::SerializeBinaryBulkStatePtr state; + type->serializeBinaryBulkStatePrefix(settings, state); + type->serializeBinaryBulkWithMultipleStreams(*columns[i], 0, size(), settings, state); + type->serializeBinaryBulkStateSuffix(settings, state); } } @@ -66,11 +75,21 @@ void MergeTreeSetIndexGranule::deserializeBinary(ReadBuffer & istr) size_type->deserializeBinary(field_rows, istr); size_t rows_to_read = field_rows.get(); + if (rows_to_read == 0) + return; + for (size_t i = 0; i < index.columns.size(); ++i) { const auto & type = index.data_types[i]; auto new_column = type->createColumn(); - type->deserializeBinaryBulk(*new_column, istr, rows_to_read, 0); + + IDataType::DeserializeBinaryBulkSettings settings; + settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return &istr; }; + settings.position_independent_encoding = false; + + IDataType::DeserializeBinaryBulkStatePtr state; + type->deserializeBinaryBulkStatePrefix(settings, state); + type->deserializeBinaryBulkWithMultipleStreams(*new_column, rows_to_read, settings, state); block.insert(ColumnWithTypeAndName(new_column->getPtr(), type, index.columns[i])); } From 4b18b7dcdc23cf7f9c64d824974bd6de95600c09 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Mar 2019 14:47:40 +0300 Subject: [PATCH 18/41] Support Nullable columns for Set index. #4486 --- .../MergeTree/MergeTreeSetSkippingIndex.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index 686d36b2c5b..8c889fbdea7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -196,10 +196,24 @@ bool SetIndexCondition::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx_granule) Block result = granule->getElementsBlock(); actions->execute(result); - const auto & column = result.getByName(expression_ast->getColumnName()).column; + auto column = result.getByName(expression_ast->getColumnName()).column->convertToFullColumnIfLowCardinality(); + auto * col_uint8 = typeid_cast(column.get()); + + const NullMap * null_map = nullptr; + + if (auto * col_nullable = typeid_cast(column.get())) + { + col_uint8 = typeid_cast(&col_nullable->getNestedColumn()); + null_map = &col_nullable->getNullMapData(); + } + + if (!col_uint8) + throw Exception("ColumnUInt8 expected as Set index condition result.", ErrorCodes::LOGICAL_ERROR); + + auto & condition = col_uint8->getData(); for (size_t i = 0; i < column->size(); ++i) - if (column->getInt(i) & 1) + if ((!null_map || (*null_map)[i] == 0) && condition[i] & 1) return true; return false; From 04bfc4743b398ea6556804fa46acd5a0c7769e9a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Mar 2019 15:13:49 +0300 Subject: [PATCH 19/41] Added tests for Set index with Nullable and LowCardinality types. #4486 --- ...ith_nullable_and_low_cardinality.reference | 30 ++++++++ ...ndex_with_nullable_and_low_cardinality.sql | 69 +++++++++++++++++++ ...nullable_and_low_cardinality_bug.reference | 1 + ..._with_nullable_and_low_cardinality_bug.sql | 20 ++++++ 4 files changed, 120 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.reference create mode 100644 dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.sql create mode 100644 dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.reference create mode 100644 dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql diff --git a/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.reference b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.reference new file mode 100644 index 00000000000..f3918a338c4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.reference @@ -0,0 +1,30 @@ +1 a +- +2 b +- +-- +1 a +- +2 b +- +-- +1 a +- +2 b +- +---- +1 a +- +2 b +- +-- +1 a +- +2 b +- +-- +1 a +- +2 b +- +---- diff --git a/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.sql b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.sql new file mode 100644 index 00000000000..363e0e9c0b5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality.sql @@ -0,0 +1,69 @@ +SET allow_experimental_data_skipping_indices=1; + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b Nullable(String), INDEX b_index b TYPE set(0) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'); +insert into test.nullable_set_index values (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '--'; + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b Nullable(String), INDEX b_index b TYPE set(1) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'); +insert into test.nullable_set_index values (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '--'; + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b Nullable(String), INDEX b_index b TYPE set(0) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'), (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '----'; + + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b LowCardinality(Nullable(String)), INDEX b_index b TYPE set(0) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'); +insert into test.nullable_set_index values (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '--'; + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b LowCardinality(Nullable(String)), INDEX b_index b TYPE set(1) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'); +insert into test.nullable_set_index values (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '--'; + +drop table if exists test.nullable_set_index; +create table test.nullable_set_index (a UInt64, b LowCardinality(Nullable(String)), INDEX b_index b TYPE set(0) GRANULARITY 8192) engine = MergeTree order by a; +insert into test.nullable_set_index values (1, 'a'), (2, 'b'); +select * from test.nullable_set_index where b = 'a'; +select '-'; +select * from test.nullable_set_index where b = 'b'; +select '-'; +select * from test.nullable_set_index where b = 'c'; +select '----'; + +drop table if exists test.nullable_set_index; + diff --git a/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.reference b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.reference new file mode 100644 index 00000000000..051c3fcdbcf --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.reference @@ -0,0 +1 @@ +subscribe alice diff --git a/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql new file mode 100644 index 00000000000..5bec1210a1e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql @@ -0,0 +1,20 @@ +SET allow_experimental_data_skipping_indices=1; + +drop table if exists test.null_lc_set_index; + +CREATE TABLE test.null_lc_set_index ( + timestamp DateTime, + action LowCardinality(Nullable(String)), + user LowCardinality(Nullable(String)), + INDEX test_user_idx (user) TYPE set(0) GRANULARITY 8192 +) ENGINE=MergeTree + PARTITION BY toYYYYMMDD(timestamp) + ORDER BY (timestamp, action, cityHash64(user)) + SAMPLE BY cityHash64(user); +INSERT INTO test.null_lc_set_index VALUES (1550883010, 'subscribe', 'alice'); +INSERT INTO test.null_lc_set_index VALUES (1550883020, 'follow', 'bob'); + +SELECT action, user FROM test.null_lc_set_index WHERE user = 'alice'; + +drop table if exists test.null_lc_set_index; + From 3bcab129661120869df892851394a1a1af135be2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Mar 2019 19:52:29 +0300 Subject: [PATCH 20/41] Fix test. --- dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp index 8c889fbdea7..227c6bfd7f7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSetSkippingIndex.cpp @@ -18,7 +18,7 @@ namespace ErrorCodes } /// 0b11 -- can be true and false at the same time -const Field UNKNOWN_FIELD(3); +const Field UNKNOWN_FIELD(3u); MergeTreeSetIndexGranule::MergeTreeSetIndexGranule(const MergeTreeSetSkippingIndex & index) From 224839a58a1df77a937e3e234e3ecae344e6aef5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Mar 2019 00:53:16 +0300 Subject: [PATCH 21/41] Attempt to read the code #4475 --- dbms/src/DataTypes/DataTypeArray.cpp | 9 ++ dbms/src/DataTypes/DataTypeArray.h | 3 + .../Functions/arrayEnumerateDenseRanked.cpp | 1 + dbms/src/Functions/arrayEnumerateRanked.cpp | 88 ++++++++----------- dbms/src/Functions/arrayEnumerateRanked.h | 6 +- 5 files changed, 53 insertions(+), 54 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index c1b19d506fe..82de731f4ad 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -498,6 +498,15 @@ bool DataTypeArray::equals(const IDataType & rhs) const } +size_t DataTypeArray::getNumberOfDimensions() const +{ + const DataTypeArray * nested_array = typeid_cast(nested.get()); + if (!nested_array) + return 1; + return 1 + nested_array->getNumberOfDimensions(); /// Every modern C++ compiler optimizes tail recursion. +} + + static DataTypePtr create(const ASTPtr & arguments) { if (!arguments || arguments->children.size() != 1) diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index 58ea41236f9..1451f27dfbe 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -112,6 +112,9 @@ public: } const DataTypePtr & getNestedType() const { return nested; } + + /// 1 for plain array, 2 for array of arrays and so on. + size_t getNumberOfDimensions() const; }; } diff --git a/dbms/src/Functions/arrayEnumerateDenseRanked.cpp b/dbms/src/Functions/arrayEnumerateDenseRanked.cpp index 05e597e3e73..cc81e35cbe7 100644 --- a/dbms/src/Functions/arrayEnumerateDenseRanked.cpp +++ b/dbms/src/Functions/arrayEnumerateDenseRanked.cpp @@ -4,6 +4,7 @@ namespace DB { + class FunctionArrayEnumerateDenseRanked : public FunctionArrayEnumerateRankedExtended { using Base = FunctionArrayEnumerateRankedExtended; diff --git a/dbms/src/Functions/arrayEnumerateRanked.cpp b/dbms/src/Functions/arrayEnumerateRanked.cpp index ff95344dd57..8341bae97bc 100644 --- a/dbms/src/Functions/arrayEnumerateRanked.cpp +++ b/dbms/src/Functions/arrayEnumerateRanked.cpp @@ -1,61 +1,53 @@ +#include +#include #include "arrayEnumerateRanked.h" + namespace DB { + ArraysDepths getArraysDepths(const ColumnsWithTypeAndName & arguments) { const size_t num_arguments = arguments.size(); + DepthType clear_depth = 1; - DepthType max_array_depth = 0; DepthTypes depths; + /// function signature is the following: + /// f(c0, arr1, c1, arr2, c2, ...) + /// + /// c0 is something called "clear_depth" here. + /// cN... - how deep to look into the corresponding arrN, (called "depths" here) + /// may be omitted - then it means "look at the full depth". + size_t array_num = 0; - DepthType last_array_depth = 0; + DepthType prev_array_depth = 0; for (size_t i = 0; i < num_arguments; ++i) { - const auto type = arguments[i].type; + const DataTypePtr & type = arguments[i].type; + const DataTypeArray * type_array = typeid_cast(type.get()); - if (isArray(type)) + if (type_array) { - if (depths.size() < array_num && last_array_depth) + if (depths.size() < array_num && prev_array_depth) { - depths.emplace_back(last_array_depth); - last_array_depth = 0; + depths.emplace_back(prev_array_depth); + prev_array_depth = 0; } - DepthType depth = 0; - auto sub_type = type; - do - { - auto sub_type_array = typeid_cast(sub_type.get()); - if (!sub_type_array) - break; - sub_type = sub_type_array->getNestedType(); - ++depth; - } while (isArray(sub_type)); - last_array_depth = depth; + prev_array_depth = type_array->getNumberOfDimensions(); ++array_num; } - - if (!arguments[i].column) - continue; - - const IColumn * non_const = nullptr; - if (auto const_array_column = typeid_cast(arguments[i].column.get())) - non_const = const_array_column->getDataColumnPtr().get(); - const auto array = typeid_cast(non_const ? non_const : arguments[i].column.get()); - - if (!array) + else { const auto & depth_column = arguments[i].column; if (depth_column && depth_column->isColumnConst()) { - auto value = depth_column->getUInt(0); + UInt64 value = static_cast(*depth_column).getValue(); if (!value) - throw Exception( - "Arguments for function arrayEnumerateUniqRanked/arrayEnumerateDenseRanked incorrect: depth (" - + std::to_string(value) + ") cant be 0.", + throw Exception("Incorrect arguments for function arrayEnumerateUniqRanked or arrayEnumerateDenseRanked: depth (" + + std::to_string(value) + ") cannot be 0.", ErrorCodes::BAD_ARGUMENTS); if (i == 0) @@ -65,38 +57,30 @@ ArraysDepths getArraysDepths(const ColumnsWithTypeAndName & arguments) else { if (depths.size() >= array_num) - { - throw Exception( - "Arguments for function arrayEnumerateUniqRanked/arrayEnumerateDenseRanked incorrect: depth (" - + std::to_string(value) + ") for missing array.", + throw Exception("Incorrect arguments for function arrayEnumerateUniqRanked or arrayEnumerateDenseRanked: depth (" + + std::to_string(value) + ") for missing array.", ErrorCodes::BAD_ARGUMENTS); - } + depths.emplace_back(value); } } } } + if (depths.size() < array_num) - { - depths.emplace_back(last_array_depth); - } - - - for (auto & depth : depths) - { - if (max_array_depth < depth) - max_array_depth = depth; - } + depths.emplace_back(prev_array_depth); if (depths.empty()) - throw Exception( - "Arguments for function arrayEnumerateUniqRanked/arrayEnumerateDenseRanked incorrect: At least one array should be passed.", + throw Exception("Incorrect arguments for function arrayEnumerateUniqRanked or arrayEnumerateDenseRanked: at least one array should be passed.", ErrorCodes::BAD_ARGUMENTS); + DepthType max_array_depth = 0; + for (auto depth : depths) + max_array_depth = std::max(depth, max_array_depth); + if (clear_depth > max_array_depth) - throw Exception( - "Arguments for function arrayEnumerateUniqRanked/arrayEnumerateDenseRanked incorrect: clear_depth (" - + std::to_string(clear_depth) + ") cant be larger than max_array_depth (" + std::to_string(max_array_depth) + ").", + throw Exception("Incorrect arguments for function arrayEnumerateUniqRanked or arrayEnumerateDenseRanked: clear_depth (" + + std::to_string(clear_depth) + ") cant be larger than max_array_depth (" + std::to_string(max_array_depth) + ").", ErrorCodes::BAD_ARGUMENTS); return {clear_depth, depths, max_array_depth}; diff --git a/dbms/src/Functions/arrayEnumerateRanked.h b/dbms/src/Functions/arrayEnumerateRanked.h index 8e87d38f025..1fd6906a8db 100644 --- a/dbms/src/Functions/arrayEnumerateRanked.h +++ b/dbms/src/Functions/arrayEnumerateRanked.h @@ -33,6 +33,7 @@ struct ArraysDepths DepthTypes depths; DepthType max_array_depth; }; + /// Return depth info about passed arrays ArraysDepths getArraysDepths(const ColumnsWithTypeAndName & arguments); @@ -55,7 +56,7 @@ public: + ", should be at least 1.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - const auto & arrays_depths = getArraysDepths(arguments); + const ArraysDepths arrays_depths = getArraysDepths(arguments); DataTypePtr type = std::make_shared(); for (DepthType i = 0; i < arrays_depths.max_array_depth; ++i) @@ -113,7 +114,8 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( const auto & arrays_depths = getArraysDepths(args); - auto get_array_column = [&](const auto & column) -> const DB::ColumnArray * { + auto get_array_column = [&](const auto & column) -> const DB::ColumnArray * + { const ColumnArray * array = checkAndGetColumn(column); if (!array) { From b5b3f52f0f6cf865e3c91f0ac02f75f4d3f30589 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Mar 2019 01:43:18 +0300 Subject: [PATCH 22/41] Added documentation --- dbms/src/Functions/arrayEnumerateRanked.h | 53 ++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/dbms/src/Functions/arrayEnumerateRanked.h b/dbms/src/Functions/arrayEnumerateRanked.h index 1fd6906a8db..6e825901260 100644 --- a/dbms/src/Functions/arrayEnumerateRanked.h +++ b/dbms/src/Functions/arrayEnumerateRanked.h @@ -12,6 +12,47 @@ #include +/** The function will enumerate distinct values of the passed multidimensional arrays looking inside at the specified depths. + * This is very unusual function made as a special order for Yandex.Metrica. + * + * arrayEnumerateUniqRanked(['hello', 'world', 'hello']) = [1, 1, 2] + * - it returns similar structured array containing number of occurence of the corresponding value. + * + * arrayEnumerateUniqRanked([['hello', 'world'], ['hello'], ['hello']], 1) = [1, 1, 2] + * - look at the depth 1 by default. Elements are ['hello', 'world'], ['hello'], ['hello']. + * + * arrayEnumerateUniqRanked([['hello', 'world'], ['hello'], ['hello']]) = [[1,1],[2],[3]] + * - look at the depth 2. Return similar structured array. + * arrayEnumerateUniqRanked([['hello', 'world'], ['hello'], ['hello']], 2) = [[1,1],[2],[3]] + * - look at the maximum depth by default. + * + * We may pass multiple array arguments. Their elements will be processed as zipped to tuple. + * + * arrayEnumerateUniqRanked(['hello', 'hello', 'world', 'world'], ['a', 'b', 'b', 'b']) = [1, 1, 1, 2] + * + * We may provide arrays of different depths to look at different arguments. + * + * arrayEnumerateUniqRanked([['hello', 'world'], ['hello'], ['world'], ['world']], ['a', 'b', 'b', 'b']) = [[1,1],[1],[1],[2]] + * arrayEnumerateUniqRanked([['hello', 'world'], ['hello'], ['world'], ['world']], 1, ['a', 'b', 'b', 'b'], 1) = [1, 1, 1, 2] + * + * When depths are different, we process less deep arrays as promoted to deeper arrays of similar structure by duplicating elements. + * + * arrayEnumerateUniqRanked( + * [['hello', 'world'], ['hello'], ['world'], ['world']], + * ['a', 'b', 'b', 'b']) + * = arrayEnumerateUniqRanked( + * [['hello', 'world'], ['hello'], ['world'], ['world']], + * [['a', 'a'], ['b'], ['b'], ['b']]) + * + * Finally, we can provide extra first argument named "clear_depth" (it can be considered as 1 by default). + * Array elements at the clear_depth will be enumerated as separate elements (enumeration counter is reset for each new element). + * + * SELECT arrayEnumerateUniqRanked(1, [['hello', 'world'], ['hello'], ['world'], ['world']]) = [[1,1],[2],[2],[3]] + * SELECT arrayEnumerateUniqRanked(2, [['hello', 'world'], ['hello'], ['world'], ['world']]) = [[1,1],[1],[1],[1]] + * SELECT arrayEnumerateUniqRanked(1, [['hello', 'world', 'hello'], ['hello'], ['world'], ['world']]) = [[1,1,2],[3],[2],[3]] + * SELECT arrayEnumerateUniqRanked(2, [['hello', 'world', 'hello'], ['hello'], ['world'], ['world']]) = [[1,1,2],[1],[1],[1]] + */ + namespace DB { namespace ErrorCodes @@ -27,10 +68,18 @@ class FunctionArrayEnumerateDenseRanked; using DepthType = uint32_t; using DepthTypes = std::vector; + struct ArraysDepths { + /// Enumerate elements at the specified level separately. DepthType clear_depth; + + /// Effective depth is the array depth by default or lower value, specified as a constant argument following the array. + /// f([[1, 2], [3]]) - effective depth is 2. + /// f([[1, 2], [3]], 1) - effective depth is 1. DepthTypes depths; + + /// Maximum effective depth. DepthType max_array_depth; }; @@ -58,6 +107,8 @@ public: const ArraysDepths arrays_depths = getArraysDepths(arguments); + /// Return type is the array of the depth as the maximum effective depth of arguments, containing UInt32. + DataTypePtr type = std::make_shared(); for (DepthType i = 0; i < arrays_depths.max_array_depth; ++i) type = std::make_shared(type); @@ -112,7 +163,7 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( for (size_t i = 0; i < arguments.size(); ++i) args.emplace_back(block.getByPosition(arguments[i])); - const auto & arrays_depths = getArraysDepths(args); + const ArraysDepths arrays_depths = getArraysDepths(args); auto get_array_column = [&](const auto & column) -> const DB::ColumnArray * { From 5a644f0052c5568aa8bb99a30335d202fef15558 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Mar 2019 02:05:47 +0300 Subject: [PATCH 23/41] Miscellaneous --- dbms/src/Functions/arrayEnumerateRanked.h | 31 +++++++++++++---------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/dbms/src/Functions/arrayEnumerateRanked.h b/dbms/src/Functions/arrayEnumerateRanked.h index 6e825901260..9056a5cff98 100644 --- a/dbms/src/Functions/arrayEnumerateRanked.h +++ b/dbms/src/Functions/arrayEnumerateRanked.h @@ -131,15 +131,15 @@ private: /// Hash a set of keys into a UInt128 value. -static inline UInt128 ALWAYS_INLINE hash128depths(const std::vector & indexes, const ColumnRawPtrs & key_columns) +static inline UInt128 ALWAYS_INLINE hash128depths(const std::vector & indices, const ColumnRawPtrs & key_columns) { UInt128 key; SipHash hash; for (size_t j = 0, keys_size = key_columns.size(); j < keys_size; ++j) { - // Debug: const auto & field = (*key_columns[j])[indexes[j]]; DUMP(j, indexes[j], field); - key_columns[j]->updateHashWithValue(indexes[j], hash); + // Debug: const auto & field = (*key_columns[j])[indices[j]]; DUMP(j, indices[j], field); + key_columns[j]->updateHashWithValue(indices[j], hash); } hash.get128(key.low, key.high); @@ -165,6 +165,7 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( const ArraysDepths arrays_depths = getArraysDepths(args); + /// If the column is Array - return it. If the const Array - materialize it, keep ownership and return. auto get_array_column = [&](const auto & column) -> const DB::ColumnArray * { const ColumnArray * array = checkAndGetColumn(column); @@ -199,7 +200,7 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( if (*offsets_by_depth[0] != array->getOffsets()) { throw Exception( - "Lengths and depths of all arrays passed to " + getName() + " must be equal.", + "Lengths and effective depths of all arrays passed to " + getName() + " must be equal.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } } @@ -223,7 +224,7 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( if (*offsets_by_depth[col_depth] != array->getOffsets()) { throw Exception( - "Lengths and depths of all arrays passed to " + getName() + " must be equal.", + "Lengths and effective depths of all arrays passed to " + getName() + " must be equal.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } } @@ -233,7 +234,7 @@ void FunctionArrayEnumerateRankedExtended::executeImpl( { throw Exception( getName() + ": Passed array number " + std::to_string(array_num) + " depth (" - + std::to_string(arrays_depths.depths[array_num]) + ") more than actual array depth (" + std::to_string(col_depth) + + std::to_string(arrays_depths.depths[array_num]) + ") is more than the actual array depth (" + std::to_string(col_depth) + ").", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } @@ -304,6 +305,7 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( const ArraysDepths & arrays_depths, ColumnUInt32::Container & res_values) { + /// Offsets at the depth we want to look. const size_t current_offset_depth = arrays_depths.max_array_depth; const auto & offsets = *offsets_by_depth[current_offset_depth - 1]; @@ -317,22 +319,24 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>; Map indices; - std::vector indexes_by_depth(arrays_depths.max_array_depth); + std::vector indices_by_depth(arrays_depths.max_array_depth); std::vector current_offset_n_by_depth(arrays_depths.max_array_depth); UInt32 rank = 0; - std::vector columns_indexes(columns.size()); + std::vector columns_indices(columns.size()); + for (size_t off : offsets) { bool want_clear = false; + /// For each element at the depth we want to look. for (size_t j = prev_off; j < off; ++j) { for (size_t col_n = 0; col_n < columns.size(); ++col_n) - columns_indexes[col_n] = indexes_by_depth[arrays_depths.depths[col_n] - 1]; + columns_indices[col_n] = indices_by_depth[arrays_depths.depths[col_n] - 1]; - auto hash = hash128depths(columns_indexes, columns); + auto hash = hash128depths(columns_indices, columns); if constexpr (std::is_same_v) { @@ -350,13 +354,13 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( res_values[j] = idx; } - // Debug: DUMP(off, prev_off, j, columns_indexes, res_values[j], columns); + // Debug: DUMP(off, prev_off, j, columns_indices, res_values[j], columns); for (int depth = current_offset_depth - 1; depth >= 0; --depth) { - ++indexes_by_depth[depth]; + ++indices_by_depth[depth]; - if (indexes_by_depth[depth] == (*offsets_by_depth[depth])[current_offset_n_by_depth[depth]]) + if (indices_by_depth[depth] == (*offsets_by_depth[depth])[current_offset_n_by_depth[depth]]) { if (static_cast(arrays_depths.clear_depth) == depth + 1) want_clear = true; @@ -368,6 +372,7 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( } } } + if (want_clear) { want_clear = false; From dd40956d8b0b233c2c0de7c7be543fc86116a5bc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Mar 2019 02:31:34 +0300 Subject: [PATCH 24/41] Fixed test --- dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql index 0682da084c1..0557f4adf85 100644 --- a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql +++ b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql @@ -170,5 +170,5 @@ SELECT arrayEnumerateUniqRanked([1,2], 1, 2); -- { serverError 36 } SELECT arrayEnumerateUniqRanked([1,2], 1, 3, 4, 5); -- { serverError 36 } SELECT arrayEnumerateUniqRanked([1,2], 1, 3, [4], 5); -- { serverError 36 } SELECT arrayEnumerateDenseRanked([[[[[[[[[[42]]]]]]]]]]); -SELECT arrayEnumerateUniqRanked('wat', [1,2]); -- { serverError 48 } -SELECT arrayEnumerateUniqRanked(1, [1,2], 'boom'); -- { serverError 48 } +SELECT arrayEnumerateUniqRanked('wat', [1,2]); -- { serverError 170 } +SELECT arrayEnumerateUniqRanked(1, [1,2], 'boom'); -- { serverError 170 } From bc30ed92d445a49272b22821cf36d54b3b452f68 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Wed, 6 Mar 2019 15:25:59 +0300 Subject: [PATCH 25/41] fix order of destruction of table locks and child streams Incorrect order could lead to a very rare `mutex lock failed: Invalid argument` error when MergeTree table was dropped and its parts were deleted defore the destruction of streams that were reading from that parts. --- dbms/src/DataStreams/IBlockInputStream.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 05b5e059454..759f3df3e51 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -242,6 +242,10 @@ public: void enableExtremes() { enabled_extremes = true; } protected: + /// Order is important: `table_locks` must be destroyed after `children` so that tables from + /// which child streams read are protected by the locks during the lifetime of the child streams. + TableStructureReadLocks table_locks; + BlockInputStreams children; std::shared_mutex children_mutex; @@ -268,8 +272,6 @@ protected: } private: - TableStructureReadLocks table_locks; - bool enabled_extremes = false; /// The limit on the number of rows/bytes has been exceeded, and you need to stop execution on the next `read` call, as if the thread has run out. From aff14bc899bdf9788b4748481595c6518595df46 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Wed, 6 Mar 2019 16:48:59 +0300 Subject: [PATCH 26/41] clarify comment --- dbms/src/DataStreams/IBlockInputStream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.h b/dbms/src/DataStreams/IBlockInputStream.h index 759f3df3e51..8a2b1e9b148 100644 --- a/dbms/src/DataStreams/IBlockInputStream.h +++ b/dbms/src/DataStreams/IBlockInputStream.h @@ -116,7 +116,7 @@ public: */ size_t checkDepth(size_t max_depth) const { return checkDepthImpl(max_depth, max_depth); } - /// Do not allow to change the table while the blocks stream is alive. + /// Do not allow to change the table while the blocks stream and its children are alive. void addTableLock(const TableStructureReadLockPtr & lock) { table_locks.push_back(lock); } /// Get information about execution speed. From 142258f76fef7474ed93f8c620324a44db9121df Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 6 Mar 2019 19:00:07 +0300 Subject: [PATCH 27/41] Remove debian epoch from changelog --- debian/changelog.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog.in b/debian/changelog.in index 4a74214c022..ce2ee757999 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -1,4 +1,4 @@ -clickhouse (2:@VERSION_STRING@) unstable; urgency=low +clickhouse (@VERSION_STRING@) unstable; urgency=low * Modified source code From 7d0be1e1f6e424cdbfc259cad03d0814b115f0d2 Mon Sep 17 00:00:00 2001 From: Vasily Vasilkov Date: Wed, 6 Mar 2019 20:32:19 +0400 Subject: [PATCH 28/41] Add one more Java/Kotlin library for ClickHouse --- docs/en/interfaces/third-party/client_libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/interfaces/third-party/client_libraries.md b/docs/en/interfaces/third-party/client_libraries.md index 5878d6e0200..a5b9991a418 100644 --- a/docs/en/interfaces/third-party/client_libraries.md +++ b/docs/en/interfaces/third-party/client_libraries.md @@ -32,6 +32,7 @@ - [RClickhouse](https://github.com/IMSMWU/RClickhouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) + - [clickhouse-client](https://github.com/Ecwid/clickhouse-client) - Scala - [clickhouse-scala-client](https://github.com/crobox/clickhouse-scala-client) - Kotlin From 89917ced9e43d4cd3b253e079e22c97bf39f784d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Mar 2019 19:46:05 +0300 Subject: [PATCH 29/41] Fixed undefined behaviour in ThreadPool #4572 --- dbms/src/Common/ThreadPool.h | 51 +++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/dbms/src/Common/ThreadPool.h b/dbms/src/Common/ThreadPool.h index d828de3fea2..35c3395cc64 100644 --- a/dbms/src/Common/ThreadPool.h +++ b/dbms/src/Common/ThreadPool.h @@ -133,18 +133,17 @@ public: template explicit ThreadFromGlobalPool(Function && func, Args &&... args) + : state(std::make_shared(true)) { - mutex = std::make_shared(); - - /// The function object must be copyable, so we wrap lock_guard in shared_ptr. + /// NOTE: If this will throw an exception, the descructor won't be called. GlobalThreadPool::instance().scheduleOrThrow([ - mutex = mutex, - lock = std::make_shared>(*mutex), + state = state, func = std::forward(func), args = std::make_tuple(std::forward(args)...)] { DB::ThreadStatus thread_status; std::apply(func, args); + state->finish(); }); } @@ -157,7 +156,7 @@ public: { if (joinable()) std::terminate(); - mutex = std::move(rhs.mutex); + state = std::move(rhs.state); return *this; } @@ -171,26 +170,52 @@ public: { if (!joinable()) std::terminate(); - { - std::lock_guard lock(*mutex); - } - mutex.reset(); + + state->wait(); + state.reset(); } void detach() { if (!joinable()) std::terminate(); - mutex.reset(); + state.reset(); } bool joinable() const { - return static_cast(mutex); + return state != nullptr; } private: - std::shared_ptr mutex; /// Object must be moveable. + /// The state used in this object and inside the thread job. + class SharedState + { + private: + bool active; + std::mutex mutex; + std::condition_variable done_event; + + public: + SharedState(bool active) : active(active) {} + + void finish() + { + { + std::lock_guard lock(mutex); + active = false; + } + done_event.notify_one(); + } + + void wait() + { + std::unique_lock lock(mutex); + done_event.wait(lock, [this]{ return !active; }); + } + }; + + std::shared_ptr state; }; From 9a56294fdf56f3a52fa1427204da7481ba79ef0f Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Wed, 6 Mar 2019 19:41:35 +0300 Subject: [PATCH 30/41] CHYT-67: introduce host context for using CH as a library. --- dbms/programs/server/HTTPHandler.cpp | 2 ++ dbms/programs/server/HTTPHandler.h | 3 +++ dbms/programs/server/TCPHandler.cpp | 10 ++++++++-- dbms/programs/server/TCPHandler.h | 3 +++ dbms/programs/server/config.xml | 4 ++-- dbms/src/Interpreters/Context.cpp | 13 +++++++++++++ dbms/src/Interpreters/Context.h | 18 ++++++++++++++++++ 7 files changed, 49 insertions(+), 4 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 6e94d59be07..763a30c1928 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -602,6 +602,8 @@ void HTTPHandler::processQuery( }); } + customizeContext(context); + executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, [&response] (const String & content_type) { response.setContentType(content_type); }, [&response] (const String & current_query_id) { response.add("Query-Id", current_query_id); }); diff --git a/dbms/programs/server/HTTPHandler.h b/dbms/programs/server/HTTPHandler.h index 58cb6bb67f1..fb6c9fb532c 100644 --- a/dbms/programs/server/HTTPHandler.h +++ b/dbms/programs/server/HTTPHandler.h @@ -28,6 +28,9 @@ public: void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + /// This method is called right before the query execution. + virtual void customizeContext(DB::Context& /* context */) {} + private: struct Output { diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 013f047ae67..1935ea87025 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -120,10 +120,11 @@ void TCPHandler::runImpl() connection_context.setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + /// Restore context of request. + query_context = connection_context; + while (1) { - /// Restore context of request. - query_context = connection_context; /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (!static_cast(*in).poll(global_settings.poll_interval * 1000000) && !server.isCancelled()) @@ -185,6 +186,8 @@ void TCPHandler::runImpl() state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. }); + customizeContext(query_context); + bool may_have_embedded_data = client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA; /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage, may_have_embedded_data); @@ -207,6 +210,9 @@ void TCPHandler::runImpl() sendLogs(); sendEndOfStream(); + /// Restore context of request. + query_context = connection_context; + query_scope.reset(); state.reset(); } diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index 19641e88d25..bd375dc5296 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -95,6 +95,9 @@ public: void run(); + /// This method is called right before the query execution. + virtual void customizeContext(DB::Context & /*context*/) {} + private: IServer & server; Poco::Logger * log; diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml index 154ebf6c35e..7a53e7185a0 100644 --- a/dbms/programs/server/config.xml +++ b/dbms/programs/server/config.xml @@ -318,10 +318,10 @@ --> - + ../../../../geo/regions_hierarchy.txt - + ../../../../geo/ - ../../../../geo/regions_hierarchy.txt + - ../../../../geo/ +