From 455ca955e569b5e6a23605c828a90686f1ca4567 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 28 Feb 2019 14:20:27 +0300 Subject: [PATCH 1/4] DOCAPI-5203: Added the descriptions for direct I/O settings for MergeTree merge and SELECT queries. --- dbms/src/Interpreters/Settings.h | 2 +- .../en/operations/server_settings/settings.md | 2 +- docs/en/operations/settings/settings.md | 28 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 42e60a2353e..0b0f630b49f 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -103,7 +103,7 @@ struct Settings \ M(SettingUInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ") \ \ - M(SettingUInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for input/output operations is bypassing the page cache. 0 - disabled.") \ + M(SettingUInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for reading the data with O_DIRECT option during SELECT queries execution. 0 - disabled.") \ \ M(SettingBool, force_index_by_date, 0, "Throw an exception if there is a partition key in a table, and it is not used.") \ M(SettingBool, force_primary_key, 0, "Throw an exception if there is primary key in a table, and it is not used.") \ diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index f339fb6ce28..db09323bab9 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -417,7 +417,7 @@ The value 0 means that you can delete all tables without any restrictions. ## merge_tree {#server_settings-merge_tree} -Fine tuning for tables in the [ MergeTree](../../operations/table_engines/mergetree.md). +Fine tuning for tables in the [MergeTree](../../operations/table_engines/mergetree.md). For more information, see the MergeTreeSettings.h header file. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e30758fc943..2fa8f37e404 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -175,6 +175,34 @@ Any positive integer. **Default value**: 1048576. +## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} + +The threshold for using direct I/O interface of the storage disk. + +ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. + +**Possible values** + +Positive integer. + +0 — The direct I/O is disabled. + +**Default value**: `10 * 1024 * 1024 * 1024` bytes. + +## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} + +The threshold for using direct I/O interface of the storage disk. + +ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, than ClickHouse reads the data from the storage disk directly with `O_DIRECT` option. + +**Possible values** + +Positive integer. + +0 — The direct I/O is disabled. + +**Default value**: 0. + ## log_queries Setting up query logging. From 646ba6726f3b5c48fe1065a61f8e31d21342b836 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 17:37:56 +0300 Subject: [PATCH 2/4] DOCAPI-5203: Clarification of direct I/O settings descriptions. --- docs/en/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2fa8f37e404..ff31b267d1d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -177,7 +177,7 @@ Any positive integer. ## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} -The threshold for using direct I/O interface of the storage disk. +The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. @@ -191,9 +191,9 @@ Positive integer. ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} -The threshold for using direct I/O interface of the storage disk. +The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. -ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, than ClickHouse reads the data from the storage disk directly with `O_DIRECT` option. +ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, then ClickHouse reads the data from the storage disk with `O_DIRECT` option. **Possible values** From 4bae599fcdcac0394917c26d1d201fcd4a8c9479 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Mon, 4 Mar 2019 18:34:35 +0300 Subject: [PATCH 3/4] DOCAPI-5203: The direct I/O settings are moved to right places. --- docs/en/operations/settings/settings.md | 14 -------------- docs/en/operations/table_engines/mergetree.md | 2 ++ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index ff31b267d1d..8f3e0d525ef 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -175,20 +175,6 @@ Any positive integer. **Default value**: 1048576. -## min_merge_bytes_to_use_direct_io {#settings-min_merge_bytes_to_use_direct_io} - -The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. - -ClickHouse uses this setting when merging data parts of the [MergeTree](../table_engines/mergetree.md)-family tables. During the merge, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, than ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. - -**Possible values** - -Positive integer. - -0 — The direct I/O is disabled. - -**Default value**: `10 * 1024 * 1024 * 1024` bytes. - ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 7d86776d27b..3658392c70c 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -70,6 +70,8 @@ For a description of request parameters, see [request description](../../query_l - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + **Example of sections setting** From 16c967002e528644963a8600cd8a97458c34005a Mon Sep 17 00:00:00 2001 From: BayoNet Date: Tue, 5 Mar 2019 13:16:45 +0300 Subject: [PATCH 4/4] DOCAPI-5203: Fix in terms of direct I/O. --- docs/en/operations/settings/settings.md | 2 +- docs/en/operations/table_engines/mergetree.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8f3e0d525ef..fff660999e2 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -177,7 +177,7 @@ Any positive integer. ## min_bytes_to_use_direct_io {#settings-min_bytes_to_use_direct_io} -The minimum data volume to be read from storage required for using of the direct I/O interface of the storage disk. +The minimum data volume to be read from storage required for using of the direct I/O access to the storage disk. ClickHouse uses this setting when selecting the data from tables. If summary storage volume of all the data to be read exceeds `min_bytes_to_use_direct_io` bytes, then ClickHouse reads the data from the storage disk with `O_DIRECT` option. diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index 3658392c70c..53bb909b16d 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -70,7 +70,7 @@ For a description of request parameters, see [request description](../../query_l - `SETTINGS` — Additional parameters that control the behavior of the `MergeTree`: - `index_granularity` — The granularity of an index. The number of data rows between the "marks" of an index. By default, 8192. The list of all available parameters you can see in [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O interface of the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. + - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation required for using of the direct I/O access to the storage disk. During the merging of the data parts, ClickHouse calculates summary storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, thеn ClickHouse reads and writes the data using direct I/O interface (`O_DIRECT` option) to the storage disk. If `min_merge_bytes_to_use_direct_io = 0`, then the direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. **Example of sections setting**