From 91e78672fa3fb3b74c233516ae85a7ecd89b7c14 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 24 Jun 2020 15:19:29 +0300 Subject: [PATCH 001/390] Enable compact parts by default for small parts --- src/Interpreters/SystemLog.cpp | 3 +-- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index b432cd8803b..c2e82646de9 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -56,8 +56,7 @@ std::shared_ptr createSystemLog( else { String partition_by = config.getString(config_prefix + ".partition_by", "toYYYYMM(event_date)"); - engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time)" - "SETTINGS min_bytes_for_wide_part = '10M'"; /// Use polymorphic parts for log tables by default + engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time)"; } size_t flush_interval_milliseconds = config.getUInt64(config_prefix + ".flush_interval_milliseconds", DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS); diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index f2d2a7cc3d4..d88fe26454c 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -29,7 +29,7 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ \ /** Data storing format settings. */ \ - M(SettingUInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ + M(SettingUInt64, min_bytes_for_wide_part, 10485760, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ M(SettingUInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \ \ /** Merge settings. */ \ From c5f46b37e6a044c3a690678c7542f749a265a546 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 28 Jun 2020 00:18:27 +0300 Subject: [PATCH 002/390] Update some tests but not all --- ...system_columns_and_system_tables.reference | 2 +- .../00804_test_delta_codec_compression.sql | 12 ++++----- ...ndex_granularity_collapsing_merge_tree.sql | 18 +++++++------ ..._adaptive_index_granularity_merge_tree.sql | 25 +++++++++++-------- ...index_granularity_replacing_merge_tree.sql | 10 ++++---- ...larity_versioned_collapsing_merge_tree.sql | 6 ++--- ...test_fix_extra_seek_on_compressed_cache.sh | 2 +- .../queries/0_stateless/00933_ttl_simple.sql | 10 +------- ...hecksums_in_system_parts_columns_table.sql | 12 ++++----- .../0_stateless/01039_mergetree_exec_time.sql | 2 +- ...1042_check_query_and_last_granule_size.sql | 4 +-- .../01045_order_by_pk_special_storages.sh | 12 ++++----- .../0_stateless/01055_compact_parts.sql | 3 ++- .../00152_insert_different_granularity.sql | 4 +-- 14 files changed, 61 insertions(+), 61 deletions(-) diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference b/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference index 1d1177ba447..ff02b0ba702 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference @@ -6,7 +6,7 @@ │ name2 │ 1 │ 0 │ 0 │ 0 │ │ name3 │ 0 │ 0 │ 0 │ 0 │ └───────┴─────────────────────┴───────────────────┴───────────────────┴────────────────────┘ -231 1 +147 1 ┌─name────────────────┬─partition_key─┬─sorting_key───┬─primary_key─┬─sampling_key─┐ │ check_system_tables │ date │ date, version │ date │ │ └─────────────────────┴───────────────┴───────────────┴─────────────┴──────────────┘ diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql index ad104eff92c..91bc45df63d 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql @@ -7,12 +7,12 @@ DROP TABLE IF EXISTS default_codec_synthetic; CREATE TABLE delta_codec_synthetic ( id UInt64 Codec(Delta, ZSTD(3)) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; CREATE TABLE default_codec_synthetic ( id UInt64 Codec(ZSTD(3)) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO delta_codec_synthetic SELECT number FROM system.numbers LIMIT 5000000; INSERT INTO default_codec_synthetic SELECT number FROM system.numbers LIMIT 5000000; @@ -45,12 +45,12 @@ DROP TABLE IF EXISTS default_codec_float; CREATE TABLE delta_codec_float ( id Float64 Codec(Delta, LZ4HC) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; CREATE TABLE default_codec_float ( id Float64 Codec(LZ4HC) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO delta_codec_float SELECT number FROM numbers(1547510400, 500000) WHERE number % 3 == 0 OR number % 5 == 0 OR number % 7 == 0 OR number % 11 == 0; INSERT INTO default_codec_float SELECT * from delta_codec_float; @@ -83,12 +83,12 @@ DROP TABLE IF EXISTS default_codec_string; CREATE TABLE delta_codec_string ( id Float64 Codec(Delta, LZ4) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; CREATE TABLE default_codec_string ( id Float64 Codec(LZ4) -) ENGINE MergeTree() ORDER BY tuple(); +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO delta_codec_string SELECT concat(toString(number), toString(number % 100)) FROM numbers(1547510400, 500000); INSERT INTO default_codec_string SELECT * from delta_codec_string; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql index b610d605e23..5603b722513 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql @@ -8,10 +8,11 @@ CREATE TABLE zero_rows_per_granule ( v2 Int64, Sign Int8 ) ENGINE CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(p) ORDER BY k - SETTINGS index_granularity_bytes=20, write_final_mark = 0, - enable_vertical_merge_algorithm=1, - vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + SETTINGS index_granularity_bytes=20, write_final_mark = 0, + min_bytes_for_wide_part = 0, + enable_vertical_merge_algorithm=1, + vertical_merge_algorithm_min_rows_to_activate=0, + vertical_merge_algorithm_min_columns_to_activate=0; INSERT INTO zero_rows_per_granule (p, k, v1, v2, Sign) VALUES ('2018-05-15', 1, 1000, 2000, 1), ('2018-05-16', 2, 3000, 4000, 1), ('2018-05-17', 3, 5000, 6000, 1), ('2018-05-18', 4, 7000, 8000, 1); @@ -39,10 +40,11 @@ CREATE TABLE four_rows_per_granule ( v2 Int64, Sign Int8 ) ENGINE CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(p) ORDER BY k - SETTINGS index_granularity_bytes=110, write_final_mark = 0, - enable_vertical_merge_algorithm=1, - vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + SETTINGS index_granularity_bytes=110, write_final_mark = 0, + min_bytes_for_wide_part = 0, + enable_vertical_merge_algorithm=1, + vertical_merge_algorithm_min_rows_to_activate=0, + vertical_merge_algorithm_min_columns_to_activate=0; INSERT INTO four_rows_per_granule (p, k, v1, v2, Sign) VALUES ('2018-05-15', 1, 1000, 2000, 1), ('2018-05-16', 2, 3000, 4000, 1), ('2018-05-17', 3, 5000, 6000, 1), ('2018-05-18', 4, 7000, 8000, 1); diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql index 18ebebb316c..48b6fae19fe 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql @@ -7,7 +7,7 @@ CREATE TABLE zero_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, write_final_mark = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -34,7 +34,7 @@ CREATE TABLE two_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 40, write_final_mark = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 40, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -61,7 +61,7 @@ CREATE TABLE four_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -95,7 +95,7 @@ CREATE TABLE huge_granularity_small_blocks ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 1000000, write_final_mark = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 1000000, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -126,7 +126,7 @@ CREATE TABLE adaptive_granularity_alter ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -183,7 +183,8 @@ CREATE TABLE zero_rows_per_granule ( SETTINGS index_granularity_bytes=20, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -215,7 +216,8 @@ CREATE TABLE two_rows_per_granule ( SETTINGS index_granularity_bytes=40, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -246,7 +248,8 @@ CREATE TABLE four_rows_per_granule ( SETTINGS index_granularity_bytes = 110, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -284,7 +287,8 @@ CREATE TABLE huge_granularity_small_blocks ( SETTINGS index_granularity_bytes=1000000, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -319,7 +323,8 @@ CREATE TABLE adaptive_granularity_alter ( SETTINGS index_granularity_bytes=110, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql index f72d5f0f9cb..53a546f9d0f 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql @@ -10,7 +10,7 @@ CREATE TABLE zero_rows_per_granule ( SETTINGS index_granularity_bytes=20, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -41,7 +41,7 @@ CREATE TABLE two_rows_per_granule ( SETTINGS index_granularity_bytes=40, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -72,7 +72,7 @@ CREATE TABLE four_rows_per_granule ( SETTINGS index_granularity_bytes = 110, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -118,7 +118,7 @@ CREATE TABLE huge_granularity_small_blocks ( SETTINGS index_granularity_bytes=1000000, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -154,7 +154,7 @@ CREATE TABLE adaptive_granularity_alter ( SETTINGS index_granularity_bytes=110, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql index c5b65839b2a..05f4dc835e5 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql @@ -12,7 +12,7 @@ CREATE TABLE zero_rows_per_granule ( SETTINGS index_granularity_bytes=20, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 2, 3000, 4000, 1, 1), ('2018-05-17', 3, 5000, 6000, 1, 1), ('2018-05-18', 4, 7000, 8000, 1, 1); @@ -44,7 +44,7 @@ CREATE TABLE four_rows_per_granule ( SETTINGS index_granularity_bytes=120, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 2, 3000, 4000, 1, 1), ('2018-05-17', 3, 5000, 6000, 1, 1), ('2018-05-18', 4, 7000, 8000, 1, 1); @@ -89,7 +89,7 @@ CREATE TABLE six_rows_per_granule ( SETTINGS index_granularity_bytes=170, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0; + vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; INSERT INTO six_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 1, 1000, 2000, -1, 2); diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index 1f7571a2404..e0225f0d31d 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" -$CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt64) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY (a);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt64) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY (a) SETTINGS min_bytes_for_wide_part = 0;" $CLICKHOUSE_CLIENT --query="INSERT INTO small_table(n) SELECT * from system.numbers limit 100000;" diff --git a/tests/queries/0_stateless/00933_ttl_simple.sql b/tests/queries/0_stateless/00933_ttl_simple.sql index b924faad3f5..83d9962043d 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.sql +++ b/tests/queries/0_stateless/00933_ttl_simple.sql @@ -13,9 +13,8 @@ create table ttl_00933_1 (d DateTime, a Int, b Int) engine = MergeTree order by insert into ttl_00933_1 values (now(), 1, 2); insert into ttl_00933_1 values (now(), 3, 4); insert into ttl_00933_1 values (now() + 1000, 5, 6); +select sleep(1.1) format Null; optimize table ttl_00933_1 final; -- check ttl merge for part with both expired and unexpired values -select sleep(1.1) format Null; -- wait if very fast merge happen -optimize table ttl_00933_1 final; select a, b from ttl_00933_1; drop table if exists ttl_00933_1; @@ -24,7 +23,6 @@ create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 DAY) engine = Mer insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 3); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1 order by d; @@ -34,7 +32,6 @@ create table ttl_00933_1 (d DateTime, a Int) engine = MergeTree order by tuple() insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_1 values (toDateTime('2100-10-10 00:00:00'), 3); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1 order by d; @@ -43,7 +40,6 @@ drop table if exists ttl_00933_1; create table ttl_00933_1 (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d) ttl d + interval 1 day; insert into ttl_00933_1 values (toDate('2000-10-10'), 1); insert into ttl_00933_1 values (toDate('2100-10-10'), 2); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1 order by d; @@ -52,7 +48,6 @@ drop table if exists ttl_00933_1; create table ttl_00933_1 (b Int, a Int ttl now()-1000) engine = MergeTree order by tuple() partition by tuple(); show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1; @@ -61,7 +56,6 @@ drop table if exists ttl_00933_1; create table ttl_00933_1 (b Int, a Int ttl now()+1000) engine = MergeTree order by tuple() partition by tuple(); show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1; @@ -70,7 +64,6 @@ drop table if exists ttl_00933_1; create table ttl_00933_1 (b Int, a Int ttl today()-1) engine = MergeTree order by tuple() partition by tuple(); show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1; @@ -79,7 +72,6 @@ drop table if exists ttl_00933_1; create table ttl_00933_1 (b Int, a Int ttl today()+1) engine = MergeTree order by tuple() partition by tuple(); show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_1 final; select * from ttl_00933_1; diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index b9eed1e8183..e865ed609be 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,14 +1,14 @@ DROP TABLE IF EXISTS test_00961; -CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111); +CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111) SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); -SELECT - name, - table, - hash_of_all_files, - hash_of_uncompressed_files, +SELECT + name, + table, + hash_of_all_files, + hash_of_uncompressed_files, uncompressed_hash_of_compressed_files FROM system.parts WHERE table = 'test_00961' and database = currentDatabase(); diff --git a/tests/queries/0_stateless/01039_mergetree_exec_time.sql b/tests/queries/0_stateless/01039_mergetree_exec_time.sql index 4cefb2e9305..d3aade41cea 100644 --- a/tests/queries/0_stateless/01039_mergetree_exec_time.sql +++ b/tests/queries/0_stateless/01039_mergetree_exec_time.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS tab; -create table tab (A Int64) Engine=MergeTree order by tuple(); +create table tab (A Int64) Engine=MergeTree order by tuple() SETTINGS min_bytes_for_wide_part = 0; insert into tab select cityHash64(number) from numbers(1000); select sum(sleep(0.1)) from tab settings max_block_size = 1, max_execution_time=1; -- { serverError 159 } DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql index 9777ea1dc45..c62fe25a041 100644 --- a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql +++ b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql @@ -1,7 +1,7 @@ SET check_query_single_value_result = 0; DROP TABLE IF EXISTS check_query_test; -CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey; +CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS min_bytes_for_wide_part = 0; -- Number of rows in last granule should be equals to granularity. -- Rows in this table are short, so granularity will be 8192. @@ -17,7 +17,7 @@ DROP TABLE IF EXISTS check_query_test; DROP TABLE IF EXISTS check_query_test_non_adaptive; -CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS index_granularity_bytes = 0; +CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS index_granularity_bytes = 0, min_bytes_for_wide_part = 0; INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 81920; diff --git a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh index 0898fec802c..b0d63b75dd5 100755 --- a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh +++ b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh @@ -10,11 +10,11 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS s2" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS m" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS buf" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv" - -$CLICKHOUSE_CLIENT -q "CREATE TABLE s1 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3" -$CLICKHOUSE_CLIENT -q "CREATE TABLE s2 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3" -$CLICKHOUSE_CLIENT -q "CREATE TABLE m (a UInt32, s String) engine = Merge(currentDatabase(), 's[1,2]')" +$CLICKHOUSE_CLIENT -q "CREATE TABLE s1 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE s2 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" + +$CLICKHOUSE_CLIENT -q "CREATE TABLE m (a UInt32, s String) engine = Merge(currentDatabase(), 's[1,2]') SETTINGS min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO s1 select (number % 20) * 2 as n, toString(number * number) from numbers(100000)" $CLICKHOUSE_CLIENT -q "INSERT INTO s2 select (number % 20) * 2 + 1 as n, toString(number * number * number) from numbers(100000)" @@ -45,7 +45,7 @@ else fi $CLICKHOUSE_CLIENT -q "SELECT '---MaterializedView---'" -$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv (a UInt32, s String) engine = MergeTree ORDER BY s POPULATE AS SELECT a, s FROM s1 WHERE a % 7 = 0" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv (a UInt32, s String) engine = MergeTree ORDER BY s SETTINGS min_bytes_for_wide_part = 0 POPULATE AS SELECT a, s FROM s1 WHERE a % 7 = 0" $CLICKHOUSE_CLIENT -q "SELECT a, s FROM mv ORDER BY s LIMIT 10" rows_read=`$CLICKHOUSE_CLIENT -q "SELECT a, s FROM mv ORDER BY s LIMIT 10 FORMAT JSON" --max_threads=1 --max_block_size=20 | grep "rows_read" | sed 's/[^0-9]*//g'` @@ -59,4 +59,4 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS s1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS s2" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS m" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS buf" -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv" diff --git a/tests/queries/0_stateless/01055_compact_parts.sql b/tests/queries/0_stateless/01055_compact_parts.sql index 05b0657ba7e..e99af76439c 100755 --- a/tests/queries/0_stateless/01055_compact_parts.sql +++ b/tests/queries/0_stateless/01055_compact_parts.sql @@ -3,9 +3,10 @@ set mutations_sync = 2; drop table if exists mt_compact; create table mt_compact(a UInt64, b UInt64 DEFAULT a * a, s String, n Nested(x UInt32, y String), lc LowCardinality(String)) -engine = MergeTree +engine = MergeTree order by a partition by a % 10 settings index_granularity = 8, +min_bytes_for_wide_part = 0, min_rows_for_wide_part = 10; insert into mt_compact (a, s, n.y, lc) select number, toString((number * 2132214234 + 5434543) % 2133443), ['a', 'b', 'c'], number % 2 ? 'bar' : 'baz' from numbers(90); diff --git a/tests/queries/1_stateful/00152_insert_different_granularity.sql b/tests/queries/1_stateful/00152_insert_different_granularity.sql index 5ca34bbe48e..7e04aedf2dd 100644 --- a/tests/queries/1_stateful/00152_insert_different_granularity.sql +++ b/tests/queries/1_stateful/00152_insert_different_granularity.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS fixed_granularity_table; -CREATE TABLE fixed_granularity_table (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192, index_granularity_bytes=0; -- looks like default table before update +CREATE TABLE fixed_granularity_table (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192, index_granularity_bytes=0, min_bytes_for_wide_part = 0; -- looks like default table before update ALTER TABLE fixed_granularity_table REPLACE PARTITION 201403 FROM test.hits; @@ -29,7 +29,7 @@ ALTER TABLE test.hits ATTACH PARTITION 201403; DROP TABLE IF EXISTS hits_copy; -CREATE TABLE hits_copy (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192, index_granularity_bytes=0; +CREATE TABLE hits_copy (`WatchID` UInt64, `JavaEnable` UInt8, `Title` String, `GoodEvent` Int16, `EventTime` DateTime, `EventDate` Date, `CounterID` UInt32, `ClientIP` UInt32, `ClientIP6` FixedString(16), `RegionID` UInt32, `UserID` UInt64, `CounterClass` Int8, `OS` UInt8, `UserAgent` UInt8, `URL` String, `Referer` String, `URLDomain` String, `RefererDomain` String, `Refresh` UInt8, `IsRobot` UInt8, `RefererCategories` Array(UInt16), `URLCategories` Array(UInt16), `URLRegions` Array(UInt32), `RefererRegions` Array(UInt32), `ResolutionWidth` UInt16, `ResolutionHeight` UInt16, `ResolutionDepth` UInt8, `FlashMajor` UInt8, `FlashMinor` UInt8, `FlashMinor2` String, `NetMajor` UInt8, `NetMinor` UInt8, `UserAgentMajor` UInt16, `UserAgentMinor` FixedString(2), `CookieEnable` UInt8, `JavascriptEnable` UInt8, `IsMobile` UInt8, `MobilePhone` UInt8, `MobilePhoneModel` String, `Params` String, `IPNetworkID` UInt32, `TraficSourceID` Int8, `SearchEngineID` UInt16, `SearchPhrase` String, `AdvEngineID` UInt8, `IsArtifical` UInt8, `WindowClientWidth` UInt16, `WindowClientHeight` UInt16, `ClientTimeZone` Int16, `ClientEventTime` DateTime, `SilverlightVersion1` UInt8, `SilverlightVersion2` UInt8, `SilverlightVersion3` UInt32, `SilverlightVersion4` UInt16, `PageCharset` String, `CodeVersion` UInt32, `IsLink` UInt8, `IsDownload` UInt8, `IsNotBounce` UInt8, `FUniqID` UInt64, `HID` UInt32, `IsOldCounter` UInt8, `IsEvent` UInt8, `IsParameter` UInt8, `DontCountHits` UInt8, `WithHash` UInt8, `HitColor` FixedString(1), `UTCEventTime` DateTime, `Age` UInt8, `Sex` UInt8, `Income` UInt8, `Interests` UInt16, `Robotness` UInt8, `GeneralInterests` Array(UInt16), `RemoteIP` UInt32, `RemoteIP6` FixedString(16), `WindowName` Int32, `OpenerName` Int32, `HistoryLength` Int16, `BrowserLanguage` FixedString(2), `BrowserCountry` FixedString(2), `SocialNetwork` String, `SocialAction` String, `HTTPError` UInt16, `SendTiming` Int32, `DNSTiming` Int32, `ConnectTiming` Int32, `ResponseStartTiming` Int32, `ResponseEndTiming` Int32, `FetchTiming` Int32, `RedirectTiming` Int32, `DOMInteractiveTiming` Int32, `DOMContentLoadedTiming` Int32, `DOMCompleteTiming` Int32, `LoadEventStartTiming` Int32, `LoadEventEndTiming` Int32, `NSToDOMContentLoadedTiming` Int32, `FirstPaintTiming` Int32, `RedirectCount` Int8, `SocialSourceNetworkID` UInt8, `SocialSourcePage` String, `ParamPrice` Int64, `ParamOrderID` String, `ParamCurrency` FixedString(3), `ParamCurrencyID` UInt16, `GoalsReached` Array(UInt32), `OpenstatServiceName` String, `OpenstatCampaignID` String, `OpenstatAdID` String, `OpenstatSourceID` String, `UTMSource` String, `UTMMedium` String, `UTMCampaign` String, `UTMContent` String, `UTMTerm` String, `FromTag` String, `HasGCLID` UInt8, `RefererHash` UInt64, `URLHash` UInt64, `CLID` UInt32, `YCLID` UInt64, `ShareService` String, `ShareURL` String, `ShareTitle` String, `ParsedParams.Key1` Array(String), `ParsedParams.Key2` Array(String), `ParsedParams.Key3` Array(String), `ParsedParams.Key4` Array(String), `ParsedParams.Key5` Array(String), `ParsedParams.ValueDouble` Array(Float64), `IslandID` FixedString(16), `RequestNum` UInt32, `RequestTry` UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192, index_granularity_bytes=0, min_bytes_for_wide_part = 0; ALTER TABLE hits_copy REPLACE PARTITION 201403 FROM test.hits; From 2d43519e038ded3bef0962a3d5ff7a5da7248914 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Jul 2020 02:27:20 +0300 Subject: [PATCH 003/390] Update some tests --- .../0_stateless/00160_merge_and_index_in_in.sql | 4 ++-- .../0_stateless/00443_preferred_block_size_bytes.sh | 6 +++--- ...0484_preferred_max_column_in_block_size_bytes.sql | 8 ++++---- .../00653_verification_monotonic_data_load.sh | 12 ++++++------ ...checksums_in_system_parts_columns_table.reference | 2 +- ...00961_checksums_in_system_parts_columns_table.sql | 2 +- .../01045_order_by_pk_special_storages.sh | 2 +- .../0_stateless/01343_min_bytes_to_use_mmap_io.sql | 2 +- .../01344_min_bytes_to_use_mmap_io_index.sql | 2 +- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/queries/0_stateless/00160_merge_and_index_in_in.sql b/tests/queries/0_stateless/00160_merge_and_index_in_in.sql index 6e2838afe88..bdab3f7640d 100644 --- a/tests/queries/0_stateless/00160_merge_and_index_in_in.sql +++ b/tests/queries/0_stateless/00160_merge_and_index_in_in.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS mt_00160; DROP TABLE IF EXISTS merge_00160; -CREATE TABLE mt_00160 (d Date DEFAULT toDate('2015-05-01'), x UInt64) ENGINE = MergeTree(d, x, 1); +CREATE TABLE mt_00160 (d Date DEFAULT toDate('2015-05-01'), x UInt64) ENGINE = MergeTree PARTITION BY d ORDER BY x SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; CREATE TABLE merge_00160 (d Date, x UInt64) ENGINE = Merge(currentDatabase(), '^mt_00160$'); SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; @@ -14,7 +14,7 @@ SELECT *, b FROM merge_00160 WHERE x IN (12345, 67890) AND NOT ignore(blockSize( DROP TABLE merge_00160; DROP TABLE mt_00160; -CREATE TABLE mt_00160 (d Date DEFAULT toDate('2015-05-01'), x UInt64, y UInt64, z UInt64) ENGINE = MergeTree(d, (x, z), 1); +CREATE TABLE mt_00160 (d Date DEFAULT toDate('2015-05-01'), x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY d ORDER BY (x, z) SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; INSERT INTO mt_00160 (x, y, z) SELECT number AS x, number + 10 AS y, number / 2 AS z FROM system.numbers LIMIT 100000; diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 4bf104a2d03..c05611783bb 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" -$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO preferred_block_size_bytes (s) SELECT '16_bytes_-_-_-_' AS s FROM system.numbers LIMIT 10, 90" $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE preferred_block_size_bytes" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=26 -q "SELECT DISTINCT blockSize(), ignore(p, s) FROM preferred_block_size_bytes" @@ -17,7 +17,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" # PREWHERE using empty column $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS pbs" -$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO pbs (p, i, sa) SELECT toDate(i % 30) AS p, number AS i, ['a'] AS sa FROM system.numbers LIMIT 1000" $CLICKHOUSE_CLIENT -q "ALTER TABLE pbs ADD COLUMN s UInt8 DEFAULT 0" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=100000 -q "SELECT count() FROM pbs PREWHERE s = 0" @@ -28,7 +28,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE pbs" # Nullable PREWHERE $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" -$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO nullable_prewhere SELECT toDate(0) AS p, if(number % 2 = 0, CAST(number AS Nullable(UInt64)), CAST(NULL AS Nullable(UInt64))) AS f, number as d FROM system.numbers LIMIT 1001" $CLICKHOUSE_CLIENT -q "SELECT sum(d), sum(f), max(d) FROM nullable_prewhere PREWHERE NOT isNull(f)" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" diff --git a/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql b/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql index e3ab4180d4e..dc021ad52db 100644 --- a/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql +++ b/tests/queries/0_stateless/00484_preferred_max_column_in_block_size_bytes.sql @@ -1,5 +1,5 @@ drop table if exists tab_00484; -create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree(date, (date, x), 8192); +create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree PARTITION BY date ORDER BY (date, x) SETTINGS min_bytes_for_wide_part = 0; insert into tab_00484 select today(), number, toFixedString('', 128) from system.numbers limit 8192; set preferred_block_size_bytes = 2000000; @@ -15,19 +15,19 @@ set preferred_max_column_in_block_size_bytes = 4194304; select max(blockSize()), min(blockSize()), any(ignore(*)) from tab_00484; drop table if exists tab_00484; -create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree(date, (date, x), 32); +create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree PARTITION BY date ORDER BY (date, x) SETTINGS min_bytes_for_wide_part = 0; insert into tab_00484 select today(), number, toFixedString('', 128) from system.numbers limit 47; set preferred_max_column_in_block_size_bytes = 1152; select blockSize(), * from tab_00484 where x = 1 or x > 36 format Null; drop table if exists tab_00484; -create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree(date, (date, x), 8192); +create table tab_00484 (date Date, x UInt64, s FixedString(128)) engine = MergeTree PARTITION BY date ORDER BY (date, x) SETTINGS min_bytes_for_wide_part = 0; insert into tab_00484 select today(), number, toFixedString('', 128) from system.numbers limit 10; set preferred_max_column_in_block_size_bytes = 128; select s from tab_00484 where s == '' format Null; drop table if exists tab_00484; -create table tab_00484 (date Date, x UInt64, s String) engine = MergeTree(date, (date, x), 8192); +create table tab_00484 (date Date, x UInt64, s String) engine = MergeTree PARTITION BY date ORDER BY (date, x) SETTINGS min_bytes_for_wide_part = 0; insert into tab_00484 select today(), number, 'abc' from system.numbers limit 81920; set preferred_block_size_bytes = 0; select count(*) from tab_00484 prewhere s != 'abc' format Null; diff --git a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh index e52610f03ba..3a8c2445e24 100755 --- a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh +++ b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh @@ -20,12 +20,12 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS unsigned_integer_test_table;" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS enum_test_table;" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS date_test_table;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE string_test_table (val String) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE fixed_string_test_table (val FixedString(1)) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE signed_integer_test_table (val Int32) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE unsigned_integer_test_table (val UInt32) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE enum_test_table (val Enum16('hello' = 1, 'world' = 2, 'yandex' = 256, 'clickhouse' = 257)) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE date_test_table (val Date) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE string_test_table (val String) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fixed_string_test_table (val FixedString(1)) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE signed_integer_test_table (val Int32) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE unsigned_integer_test_table (val UInt32) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE enum_test_table (val Enum16('hello' = 1, 'world' = 2, 'yandex' = 256, 'clickhouse' = 257)) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE date_test_table (val Date) ENGINE = MergeTree ORDER BY val SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0;" ${CLICKHOUSE_CLIENT} --query="SYSTEM STOP MERGES;" diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference index 282b0ddca7b..3bcfc00eded 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference @@ -1 +1 @@ -20000101_20000101_1_1_0 test_00961 1c63ae7a38eb76e2a71c28aaf0b3ae4d 0053df9b467cc5483e752ec62e91cfd4 da96ff1e527a8a1f908ddf2b1d0af239 +20000101_20000101_1_1_0 test_00961 b78f351b7498ecc9d4732ad29c3952de 1d4b7fbf05d0fc5c2f4559ca75aa32f7 38f047b57fd1bb81cf77e273deb34218 diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index e865ed609be..f3a729dd4fd 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test_00961; -CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111) SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111); INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); diff --git a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh index b0d63b75dd5..3c549fa64ff 100755 --- a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh +++ b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh @@ -14,7 +14,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv" $CLICKHOUSE_CLIENT -q "CREATE TABLE s1 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "CREATE TABLE s2 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" -$CLICKHOUSE_CLIENT -q "CREATE TABLE m (a UInt32, s String) engine = Merge(currentDatabase(), 's[1,2]') SETTINGS min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE m (a UInt32, s String) engine = Merge(currentDatabase(), 's[1,2]')" $CLICKHOUSE_CLIENT -q "INSERT INTO s1 select (number % 20) * 2 as n, toString(number * number) from numbers(100000)" $CLICKHOUSE_CLIENT -q "INSERT INTO s2 select (number % 20) * 2 + 1 as n, toString(number * number * number) from numbers(100000)" diff --git a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql index 9ff16ca60a7..62c5d20d714 100644 --- a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql +++ b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS test_01343; -CREATE TABLE test_01343 (x String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE test_01343 (x String) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01343 VALUES ('Hello, world'); SET min_bytes_to_use_mmap_io = 1; diff --git a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql index 67baef7136d..544c0af7925 100644 --- a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql +++ b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS test_01344; -CREATE TABLE test_01344 (x String, INDEX idx (x) TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE test_01344 (x String, INDEX idx (x) TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01344 VALUES ('Hello, world'); SET min_bytes_to_use_mmap_io = 1; From 05a5a13e08f615986815e66b75a231824239f558 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 10 Aug 2020 19:23:58 +0800 Subject: [PATCH 004/390] specific ReplicatedMergeTree settings --- .../server-configuration-parameters/settings.md | 16 ++++++++++++++++ programs/server/Server.cpp | 1 + src/Interpreters/Context.cpp | 17 +++++++++++++++++ src/Interpreters/Context.h | 1 + src/Server/ReplicasStatusHandler.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 6 +++++- 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index c1ac1d0d92d..ee0373c70b4 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -521,6 +521,22 @@ For more information, see the MergeTreeSettings.h header file. ``` +## replicated\_merge\_tree {#server_configuration_parameters-replicated_merge_tree} + +Fine tuning for tables in the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/mergetree.md). + +This setting has higher priority. + +For more information, see the MergeTreeSettings.h header file. + +**Example** + +``` xml + + 5 + +``` + ## openSSL {#server_configuration_parameters-openssl} SSL client/server configuration. diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3a975325851..37228fd4a0e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -596,6 +596,7 @@ int Server::main(const std::vector & /*args*/) /// Check sanity of MergeTreeSettings on server startup global_context->getMergeTreeSettings().sanityCheck(settings); + global_context->getReplicatedMergeTreeSettings().sanityCheck(settings); /// Limit on total memory usage size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 70cf41a679c..431912711f8 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -348,6 +348,7 @@ struct ContextShared mutable std::shared_ptr merge_tree_storage_policy_selector; std::optional merge_tree_settings; /// Settings of MergeTree* engines. + std::optional replicated_merge_tree_settings; /// Settings of ReplicatedMergeTree* engines. std::atomic_size_t max_table_size_to_drop = 50000000000lu; /// Protects MergeTree tables from accidental DROP (50GB by default) std::atomic_size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default) String format_schema_path; /// Path to a directory that contains schema files used by input formats. @@ -1823,6 +1824,22 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const return *shared->merge_tree_settings; } +const MergeTreeSettings & Context::getReplicatedMergeTreeSettings() const +{ + auto lock = getLock(); + + if (!shared->replicated_merge_tree_settings) + { + const auto & config = getConfigRef(); + MergeTreeSettings mt_settings; + mt_settings.loadFromConfig("merge_tree", config); + mt_settings.loadFromConfig("replicated_merge_tree", config); + shared->replicated_merge_tree_settings.emplace(mt_settings); + } + + return *shared->replicated_merge_tree_settings; +} + const StorageS3Settings & Context::getStorageS3Settings() const { #if !defined(ARCADIA_BUILD) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index c8d13baa9ae..609440e5602 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -537,6 +537,7 @@ public: std::shared_ptr getPartLog(const String & part_database); const MergeTreeSettings & getMergeTreeSettings() const; + const MergeTreeSettings & getReplicatedMergeTreeSettings() const; const StorageS3Settings & getStorageS3Settings() const; /// Prevents DROP TABLE if its size is greater than max_size (50GB by default, max_size=0 turn off this check) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index de68635d26e..bc5436f00ee 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -33,7 +33,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request /// Even if lag is small, output detailed information about the lag. bool verbose = params.get("verbose", "") == "1"; - const MergeTreeSettings & settings = context.getMergeTreeSettings(); + const MergeTreeSettings & settings = context.getReplicatedMergeTreeSettings(); bool ok = true; std::stringstream message; diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 4526b0d4f9b..6ee63dd251f 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -514,7 +514,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) StorageInMemoryMetadata metadata; metadata.columns = args.columns; - std::unique_ptr storage_settings = std::make_unique(args.context.getMergeTreeSettings()); + std::unique_ptr storage_settings; + if (replicated) + storage_settings = std::make_unique(args.context.getReplicatedMergeTreeSettings()); + else + storage_settings = std::make_unique(args.context.getMergeTreeSettings()); if (is_extended_storage_def) { From 44364a5f59fa5d55adeea210dab9186281cd4a09 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 11 Aug 2020 21:38:25 +0800 Subject: [PATCH 005/390] add tests --- .../__init__.py | 0 .../configs/config.xml | 9 +++++ .../test_replicated_merge_tree_config/test.py | 37 +++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 tests/integration/test_replicated_merge_tree_config/__init__.py create mode 100644 tests/integration/test_replicated_merge_tree_config/configs/config.xml create mode 100644 tests/integration/test_replicated_merge_tree_config/test.py diff --git a/tests/integration/test_replicated_merge_tree_config/__init__.py b/tests/integration/test_replicated_merge_tree_config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_config/configs/config.xml b/tests/integration/test_replicated_merge_tree_config/configs/config.xml new file mode 100644 index 00000000000..d760d05f1bc --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_config/configs/config.xml @@ -0,0 +1,9 @@ + + + + 100 + + + 200 + + diff --git a/tests/integration/test_replicated_merge_tree_config/test.py b/tests/integration/test_replicated_merge_tree_config/test.py new file mode 100644 index 00000000000..bcf6517782d --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_config/test.py @@ -0,0 +1,37 @@ +import pytest +from helpers.cluster import ClickHouseCluster + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", config_dir="configs", with_zookeeper=True, + ) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def drop_table(cluster): + yield + for node in cluster.instances.values(): + node.query("DROP TABLE IF EXISTS test1") + node.query("DROP TABLE IF EXISTS test2") + + +def test_replicated_merge_tree_settings(cluster): + node = cluster.instances["node"] + node.query("CREATE TABLE test1 (id Int64) ENGINE MergeTree ORDER BY id") + node.query( + "CREATE TABLE test2 (id Int64) ENGINE ReplicatedMergeTree('/clickhouse/test', 'test') ORDER BY id" + ) + + assert node.query("SHOW CREATE test1").endswith("100") + assert node.query("SHOW CREATE test2").endswith("200") From 1476a9e23642a31f32ae9dd66d87ff005e821fbd Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 23 Aug 2020 20:38:52 +0800 Subject: [PATCH 006/390] Add replicated_merge_tree_settings table --- .../System/StorageSystemMergeTreeSettings.cpp | 11 ++++++++--- .../System/StorageSystemMergeTreeSettings.h | 14 +++++++++----- src/Storages/System/attachSystemTables.cpp | 3 ++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Storages/System/StorageSystemMergeTreeSettings.cpp b/src/Storages/System/StorageSystemMergeTreeSettings.cpp index 4de600ac036..19cbf76f252 100644 --- a/src/Storages/System/StorageSystemMergeTreeSettings.cpp +++ b/src/Storages/System/StorageSystemMergeTreeSettings.cpp @@ -7,7 +7,8 @@ namespace DB { -NamesAndTypesList SystemMergeTreeSettings::getNamesAndTypes() +template +NamesAndTypesList SystemMergeTreeSettings::getNamesAndTypes() { return { {"name", std::make_shared()}, @@ -18,9 +19,11 @@ NamesAndTypesList SystemMergeTreeSettings::getNamesAndTypes() }; } -void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +template +void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - for (const auto & setting : context.getMergeTreeSettings().all()) + const auto & settings = replicated ? context.getReplicatedMergeTreeSettings().all() : context.getMergeTreeSettings().all(); + for (const auto & setting : settings) { res_columns[0]->insert(setting.getName()); res_columns[1]->insert(setting.getValueString()); @@ -30,4 +33,6 @@ void SystemMergeTreeSettings::fillData(MutableColumns & res_columns, const Conte } } +template class SystemMergeTreeSettings; +template class SystemMergeTreeSettings; } diff --git a/src/Storages/System/StorageSystemMergeTreeSettings.h b/src/Storages/System/StorageSystemMergeTreeSettings.h index ac4d9d27505..9f61fa6f780 100644 --- a/src/Storages/System/StorageSystemMergeTreeSettings.h +++ b/src/Storages/System/StorageSystemMergeTreeSettings.h @@ -11,18 +11,22 @@ namespace DB class Context; -/** implements system table "merge_tree_settings", which allows to get information about the current MergeTree settings. +/** implements system table "merge_tree_settings" and "replicated_merge_tree_settings", + * which allows to get information about the current MergeTree settings. */ -class SystemMergeTreeSettings final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +template +class SystemMergeTreeSettings final : public ext::shared_ptr_helper>, + public IStorageSystemOneBlock> { - friend struct ext::shared_ptr_helper; + friend struct ext::shared_ptr_helper>; + public: - std::string getName() const override { return "SystemMergeTreeSettings"; } + std::string getName() const override { return replicated ? "SystemReplicatedMergeTreeSettings" : "SystemMergeTreeSettings"; } static NamesAndTypesList getNamesAndTypes(); protected: - using IStorageSystemOneBlock::IStorageSystemOneBlock; + using IStorageSystemOneBlock>::IStorageSystemOneBlock; void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; }; diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 36e4e34361b..2b7ee363f05 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -82,7 +82,8 @@ void attachSystemTablesLocal(IDatabase & system_database) attach(system_database, "functions"); attach(system_database, "events"); attach(system_database, "settings"); - attach(system_database, "merge_tree_settings"); + attach>(system_database, "merge_tree_settings"); + attach>(system_database, "replicated_merge_tree_settings"); attach(system_database, "build_options"); attach(system_database, "formats"); attach(system_database, "table_functions"); From 33a65063cee4a8f1314f7f5c064cbaefcbd59269 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sun, 30 Aug 2020 22:53:22 +0300 Subject: [PATCH 007/390] Docs for the output_format_pretty_max_value_width setting (English). --- docs/en/operations/settings/settings.md | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4995c04f712..791e3023686 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1899,4 +1899,54 @@ Possible values: Default value: `120`. +## output_format_pretty_max_value_width {#output_format_pretty_max_value_width} + +Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pretty) formats. If the value width exceeds the setting, the value is cut. + +Possible values: + +- Positive integer. Type: [UInt64](../../sql-reference/data-types/int-uint.md). +- 0 — The value is cut completely. + +Default value: `10000` symbols. + +**Examples** + +Query: +```sql +SET output_format_pretty_max_value_width = 10; +SELECT range(number) FROM system.numbers LIMIT 10 FORMAT PrettyCompactNoEscapes; +``` +Result: +```text +┌─range(number)─┐ +│ [] │ +│ [0] │ +│ [0,1] │ +│ [0,1,2] │ +│ [0,1,2,3] │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +└───────────────┘ +``` + +Query with zero width: +```sql +SET output_format_pretty_max_value_width = 0; +SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; +``` +Result: +```text +┌─range(number)─┐ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +└───────────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From 045e4f8964b1939d14251240c592839f33fa7c7b Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sun, 30 Aug 2020 23:03:16 +0300 Subject: [PATCH 008/390] Minor fix. --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 791e3023686..11ab1247753 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1905,7 +1905,7 @@ Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pret Possible values: -- Positive integer. Type: [UInt64](../../sql-reference/data-types/int-uint.md). +- Positive integer. - 0 — The value is cut completely. Default value: `10000` symbols. From 23fb122818d54e18225ea71562b35d4c82b005ac Mon Sep 17 00:00:00 2001 From: Gao Qiang <30835199+dreamerfable@users.noreply.github.com> Date: Wed, 2 Sep 2020 23:27:27 +0800 Subject: [PATCH 009/390] Update replacingmergetree.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix the wrong translation of sorting key fix the wrong format make some sentences more understandable I think the title which is a name of table engine is more approriate to keeping in english 。 --- .../mergetree-family/replacingmergetree.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md index 626597eeaf0..73328015ea9 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,8 +1,8 @@ -# 替换合并树 {#replacingmergetree} +# ReplacingMergeTree {#replacingmergetree} -该引擎和[MergeTree](mergetree.md)的不同之处在于它会删除具有相同主键的重复项。 +该引擎和 [MergeTree](mergetree.md) 的不同之处在于它会删除排序键值相同的重复项。 -数据的去重只会在合并的过程中出现。合并会在未知的时间在后台进行，因此你无法预先作出计划。有一些数据可能仍未被处理。尽管你可以调用 `OPTIMIZE` 语句发起计划外的合并，但请不要指望使用它，因为 `OPTIMIZE` 语句会引发对大量数据的读和写。 +数据的去重只会在数据合并期间进行。合并会在后台一个不确定的时间进行，因此你无法预先作出计划。有一些数据可能仍未被处理。尽管你可以调用 `OPTIMIZE` 语句发起计划外的合并，但请不要依靠它，因为 `OPTIMIZE` 语句会引发对数据的大量读写。 因此，`ReplacingMergeTree` 适用于在后台清除重复的数据以节省空间，但是它不保证没有重复的数据出现。 @@ -21,19 +21,20 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [SETTINGS name=value, ...] ``` -请求参数的描述，参考[请求参数](../../../engines/table-engines/mergetree-family/replacingmergetree.md)。 +有关建表参数的描述，可参考 [创建表](../../../sql-reference/statements/create.md#create-table-query)。 -**参数** +**ReplacingMergeTree 的参数** - `ver` — 版本列。类型为 `UInt*`, `Date` 或 `DateTime`。可选参数。 - 合并的时候，`ReplacingMergeTree` 从所有具有相同主键的行中选择一行留下： - - 如果 `ver` 列未指定，选择最后一条。 - - 如果 `ver` 列已指定，选择 `ver` 值最大的版本。 + 在数据合并的时候，`ReplacingMergeTree` 从所有具有相同排序键的行中选择一行留下： + + - 如果 `ver` 列未指定，保留最后一条。 + - 如果 `ver` 列已指定，保留 `ver` 值最大的版本。 **子句** -创建 `ReplacingMergeTree` 表时，需要与创建 `MergeTree` 表时相同的[子句](mergetree.md)。 +创建 `ReplacingMergeTree` 表时，需要使用与创建 `MergeTree` 表时相同的 [子句](mergetree.md)。
From dd0feeeaa139ddce8e171adbcaf9824cfffb66b2 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Sep 2020 11:03:28 +0800 Subject: [PATCH 010/390] ISSUES-4006 support datetime with precision --- src/Core/MySQL/MySQLReplication.cpp | 34 +++++++++++++++---- src/Core/MySQL/MySQLReplication.h | 32 +++++++++++++---- .../MySQL/MaterializeMySQLSyncThread.cpp | 3 ++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 41afe3cde6a..1b5ca132eeb 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -451,7 +451,7 @@ namespace MySQLReplication UInt32 hour = readBits(val, 2, 10, 24); UInt32 minute = readBits(val, 12, 6, 24); UInt32 second = readBits(val, 18, 6, 24); - readTimeFractionalPart(payload, reinterpret_cast(&frac_part), meta); + readTimeFractionalPart(payload, frac_part, meta); if (frac_part != 0) { @@ -481,9 +481,10 @@ namespace MySQLReplication break; } case MYSQL_TYPE_DATETIME2: { - Int64 val = 0, fsp = 0; + Int64 val = 0; + UInt32 fsp = 0; readBigEndianStrict(payload, reinterpret_cast(&val), 5); - readTimeFractionalPart(payload, reinterpret_cast(&fsp), meta); + readTimeFractionalPart(payload, fsp, meta); UInt32 year_month = readBits(val, 1, 17, 40); time_t date_time = DateLUT::instance().makeDateTime( @@ -491,14 +492,35 @@ namespace MySQLReplication , readBits(val, 23, 5, 40), readBits(val, 28, 6, 40), readBits(val, 34, 6, 40) ); - row.push_back(Field{UInt32(date_time)}); + if (!meta) + row.push_back(Field{UInt32(date_time)}); + else + { + DB::DecimalUtils::DecimalComponents components{ + static_cast(date_time), 0}; + + components.fractional = fsp; + row.push_back(Field(DecimalUtils::decimalFromComponents(components, meta))); + } + break; } case MYSQL_TYPE_TIMESTAMP2: { UInt32 sec = 0, fsp = 0; readBigEndianStrict(payload, reinterpret_cast(&sec), 4); - readTimeFractionalPart(payload, reinterpret_cast(&fsp), meta); - row.push_back(Field{sec}); + readTimeFractionalPart(payload, fsp, meta); + + if (!meta) + row.push_back(Field{sec}); + else + { + DB::DecimalUtils::DecimalComponents components{ + static_cast(sec), 0}; + + components.fractional = fsp; + row.push_back(Field(DecimalUtils::decimalFromComponents(components, meta))); + } + break; } case MYSQL_TYPE_NEWDECIMAL: { diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h index 7834a1b355c..aac075fae2f 100644 --- a/src/Core/MySQL/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -37,23 +37,41 @@ namespace MySQLReplication std::reverse(start, end); } - inline void readTimeFractionalPart(ReadBuffer & payload, char * to, UInt16 meta) + inline void readTimeFractionalPart(ReadBuffer & payload, UInt32 & factional, UInt16 meta) { switch (meta) { case 1: - case 2: { - readBigEndianStrict(payload, to, 1); + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 1); + factional /= 10; + break; + } + case 2: + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 1); break; } case 3: - case 4: { - readBigEndianStrict(payload, to, 2); + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 2); + factional /= 10; + break; + } + case 4: + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 2); break; } case 5: - case 6: { - readBigEndianStrict(payload, to, 3); + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 3); + factional /= 10; + break; + } + case 6: + { + readBigEndianStrict(payload, reinterpret_cast(&factional), 3); break; } default: diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index 851ea351876..7ded256dd86 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -9,6 +9,7 @@ # include # include # include +# include # include # include # include @@ -451,6 +452,8 @@ static void writeFieldsToColumn( write_data_to_column(casted_float32_column, Float64(), Float32()); else if (ColumnFloat64 * casted_float64_column = typeid_cast(&column_to)) write_data_to_column(casted_float64_column, Float64(), Float64()); + else if (ColumnDecimal * casted_date_time_64_column = typeid_cast *>(&column_to)) + write_data_to_column(casted_date_time_64_column, DateTime64(), DateTime64()); else if (ColumnInt32 * casted_int32_column = typeid_cast(&column_to)) { for (size_t index = 0; index < rows_data.size(); ++index) From 9c091fb2c1e251f4714ea4415fe33ce49a767e70 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Sep 2020 11:08:38 +0800 Subject: [PATCH 011/390] ISSUES-4006 remove unsupport data type --- src/Core/MySQL/MySQLReplication.cpp | 126 ++-------------------------- 1 file changed, 5 insertions(+), 121 deletions(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1b5ca132eeb..557b84dd9bf 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -198,10 +198,9 @@ namespace MySQLReplication case MYSQL_TYPE_LONGLONG: case MYSQL_TYPE_INT24: case MYSQL_TYPE_DATE: - case MYSQL_TYPE_TIME: case MYSQL_TYPE_DATETIME: - case MYSQL_TYPE_YEAR: - case MYSQL_TYPE_NEWDATE: { + case MYSQL_TYPE_NEWDATE: + { /// No data here. column_meta.emplace_back(0); break; @@ -211,24 +210,21 @@ namespace MySQLReplication case MYSQL_TYPE_DOUBLE: case MYSQL_TYPE_TIMESTAMP2: case MYSQL_TYPE_DATETIME2: - case MYSQL_TYPE_TIME2: - case MYSQL_TYPE_JSON: case MYSQL_TYPE_BLOB: - case MYSQL_TYPE_GEOMETRY: { + { column_meta.emplace_back(UInt16(meta[pos])); pos += 1; break; } case MYSQL_TYPE_NEWDECIMAL: - case MYSQL_TYPE_STRING: { + case MYSQL_TYPE_STRING: + { auto b0 = UInt16(meta[pos] << 8); auto b1 = UInt8(meta[pos + 1]); column_meta.emplace_back(UInt16(b0 + b1)); pos += 2; break; } - - case MYSQL_TYPE_BIT: case MYSQL_TYPE_VARCHAR: case MYSQL_TYPE_VAR_STRING: { auto b0 = UInt8(meta[pos]); @@ -405,21 +401,6 @@ namespace MySQLReplication row.push_back(Field{val}); break; } - case MYSQL_TYPE_TIME: { - UInt32 i24 = 0; - payload.readStrict(reinterpret_cast(&i24), 3); - - String time_buff; - time_buff.resize(8); - sprintf( - time_buff.data(), - "%02d:%02d:%02d", - static_cast(i24 / 10000), - static_cast(i24 % 10000) / 100, - static_cast(i24 % 100)); - row.push_back(Field{String{time_buff}}); - break; - } case MYSQL_TYPE_DATE: { UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); @@ -430,56 +411,6 @@ namespace MySQLReplication row.push_back(Field(date_day_number.toUnderType())); break; } - case MYSQL_TYPE_YEAR: { - Int32 val = 0; - payload.readStrict(reinterpret_cast(&val), 1); - - String time_buff; - time_buff.resize(4); - sprintf(time_buff.data(), "%04d", (val + 1900)); - row.push_back(Field{String{time_buff}}); - break; - } - case MYSQL_TYPE_TIME2: { - UInt32 val = 0, frac_part = 0; - - readBigEndianStrict(payload, reinterpret_cast(&val), 3); - if (readBits(val, 0, 1, 24) == 0) - { - val = ~val + 1; - } - UInt32 hour = readBits(val, 2, 10, 24); - UInt32 minute = readBits(val, 12, 6, 24); - UInt32 second = readBits(val, 18, 6, 24); - readTimeFractionalPart(payload, frac_part, meta); - - if (frac_part != 0) - { - String time_buff; - time_buff.resize(15); - sprintf( - time_buff.data(), - "%02d:%02d:%02d.%06d", - static_cast(hour), - static_cast(minute), - static_cast(second), - static_cast(frac_part)); - row.push_back(Field{String{time_buff}}); - } - else - { - String time_buff; - time_buff.resize(8); - sprintf( - time_buff.data(), - "%02d:%02d:%02d", - static_cast(hour), - static_cast(minute), - static_cast(second)); - row.push_back(Field{String{time_buff}}); - } - break; - } case MYSQL_TYPE_DATETIME2: { Int64 val = 0; UInt32 fsp = 0; @@ -607,42 +538,6 @@ namespace MySQLReplication row.push_back(Field{String{format}}); break; } - case MYSQL_TYPE_ENUM: { - Int32 val = 0; - Int32 len = (meta & 0xff); - switch (len) - { - case 1: { - payload.readStrict(reinterpret_cast(&val), 1); - break; - } - case 2: { - payload.readStrict(reinterpret_cast(&val), 2); - break; - } - default: - break; - } - row.push_back(Field{Int32{val}}); - break; - } - case MYSQL_TYPE_BIT: { - UInt32 bits = ((meta >> 8) * 8) + (meta & 0xff); - UInt32 size = (bits + 7) / 8; - - Bitmap bitmap1; - readBitmap(payload, bitmap1, size); - row.push_back(Field{UInt64{bitmap1.to_ulong()}}); - break; - } - case MYSQL_TYPE_SET: { - UInt32 size = (meta & 0xff); - - Bitmap bitmap1; - readBitmap(payload, bitmap1, size); - row.push_back(Field{UInt64{bitmap1.to_ulong()}}); - break; - } case MYSQL_TYPE_VARCHAR: case MYSQL_TYPE_VAR_STRING: { uint32_t size = 0; @@ -678,7 +573,6 @@ namespace MySQLReplication row.push_back(Field{String{val}}); break; } - case MYSQL_TYPE_GEOMETRY: case MYSQL_TYPE_BLOB: { UInt32 size = 0; switch (meta) @@ -709,16 +603,6 @@ namespace MySQLReplication row.push_back(Field{String{val}}); break; } - case MYSQL_TYPE_JSON: { - UInt32 size = 0; - payload.readStrict(reinterpret_cast(&size), meta); - - String val; - val.resize(size); - payload.readStrict(reinterpret_cast(val.data()), size); - row.push_back(Field{String{val}}); - break; - } default: throw ReplicationError( "ParseRow: Unhandled MySQL field type:" + std::to_string(field_type), ErrorCodes::UNKNOWN_EXCEPTION); From 802a5a31a0e6a9328cf5868f094c100927ca52c4 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 Sep 2020 11:11:18 +0800 Subject: [PATCH 012/390] ISSUES-4006 fix code style --- src/Core/MySQL/MySQLReplication.cpp | 45 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 557b84dd9bf..6a102dca1f3 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -352,56 +352,65 @@ namespace MySQLReplication switch (field_type) { - case MYSQL_TYPE_TINY: { + case MYSQL_TYPE_TINY: + { UInt8 val = 0; payload.readStrict(reinterpret_cast(&val), 1); row.push_back(Field{UInt8{val}}); break; } - case MYSQL_TYPE_SHORT: { + case MYSQL_TYPE_SHORT: + { UInt16 val = 0; payload.readStrict(reinterpret_cast(&val), 2); row.push_back(Field{UInt16{val}}); break; } - case MYSQL_TYPE_INT24: { + case MYSQL_TYPE_INT24: + { Int32 val = 0; payload.readStrict(reinterpret_cast(&val), 3); row.push_back(Field{Int32{val}}); break; } - case MYSQL_TYPE_LONG: { + case MYSQL_TYPE_LONG: + { UInt32 val = 0; payload.readStrict(reinterpret_cast(&val), 4); row.push_back(Field{UInt32{val}}); break; } - case MYSQL_TYPE_LONGLONG: { + case MYSQL_TYPE_LONGLONG: + { UInt64 val = 0; payload.readStrict(reinterpret_cast(&val), 8); row.push_back(Field{UInt64{val}}); break; } - case MYSQL_TYPE_FLOAT: { + case MYSQL_TYPE_FLOAT: + { Float32 val = 0; payload.readStrict(reinterpret_cast(&val), 4); row.push_back(Field{Float32{val}}); break; } - case MYSQL_TYPE_DOUBLE: { + case MYSQL_TYPE_DOUBLE: + { Float64 val = 0; payload.readStrict(reinterpret_cast(&val), 8); row.push_back(Field{Float64{val}}); break; } - case MYSQL_TYPE_TIMESTAMP: { + case MYSQL_TYPE_TIMESTAMP: + { UInt32 val = 0; payload.readStrict(reinterpret_cast(&val), 4); row.push_back(Field{val}); break; } - case MYSQL_TYPE_DATE: { + case MYSQL_TYPE_DATE: + { UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); @@ -411,7 +420,8 @@ namespace MySQLReplication row.push_back(Field(date_day_number.toUnderType())); break; } - case MYSQL_TYPE_DATETIME2: { + case MYSQL_TYPE_DATETIME2: + { Int64 val = 0; UInt32 fsp = 0; readBigEndianStrict(payload, reinterpret_cast(&val), 5); @@ -436,7 +446,8 @@ namespace MySQLReplication break; } - case MYSQL_TYPE_TIMESTAMP2: { + case MYSQL_TYPE_TIMESTAMP2: + { UInt32 sec = 0, fsp = 0; readBigEndianStrict(payload, reinterpret_cast(&sec), 4); readTimeFractionalPart(payload, fsp, meta); @@ -454,7 +465,8 @@ namespace MySQLReplication break; } - case MYSQL_TYPE_NEWDECIMAL: { + case MYSQL_TYPE_NEWDECIMAL: + { Int8 digits_per_integer = 9; Int8 precision = meta >> 8; Int8 decimals = meta & 0xff; @@ -539,7 +551,8 @@ namespace MySQLReplication break; } case MYSQL_TYPE_VARCHAR: - case MYSQL_TYPE_VAR_STRING: { + case MYSQL_TYPE_VAR_STRING: + { uint32_t size = 0; if (meta < 256) { @@ -556,7 +569,8 @@ namespace MySQLReplication row.push_back(Field{String{val}}); break; } - case MYSQL_TYPE_STRING: { + case MYSQL_TYPE_STRING: + { UInt32 size = 0; if (field_len < 256) { @@ -573,7 +587,8 @@ namespace MySQLReplication row.push_back(Field{String{val}}); break; } - case MYSQL_TYPE_BLOB: { + case MYSQL_TYPE_BLOB: + { UInt32 size = 0; switch (meta) { From 9fef663caad63f65232b4951a3451ceb55c2af76 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 3 Sep 2020 21:06:30 +0300 Subject: [PATCH 013/390] add test --- .../01457_order_by_nulls_first.reference | 0 .../01457_order_by_nulls_first.sql | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/queries/0_stateless/01457_order_by_nulls_first.reference create mode 100644 tests/queries/0_stateless/01457_order_by_nulls_first.sql diff --git a/tests/queries/0_stateless/01457_order_by_nulls_first.reference b/tests/queries/0_stateless/01457_order_by_nulls_first.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01457_order_by_nulls_first.sql b/tests/queries/0_stateless/01457_order_by_nulls_first.sql new file mode 100644 index 00000000000..7e391276d84 --- /dev/null +++ b/tests/queries/0_stateless/01457_order_by_nulls_first.sql @@ -0,0 +1,26 @@ +drop table if exists order_by_nulls_first; + +CREATE TABLE order_by_nulls_first +(diff Nullable(Int16), traf UInt64) +ENGINE = MergeTree ORDER BY tuple(); + +insert into order_by_nulls_first values (NULL,1),(NULL,0),(NULL,0),(NULL,0),(NULL,0),(NULL,0),(28,0),(0,0); + +SELECT + diff, + traf +FROM order_by_nulls_first +order by diff desc NULLS FIRST, traf +limit 1, 4; + +select '---'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff DESC NULLS FIRST, + traf ASC; + +drop table if exists order_by_nulls_first; \ No newline at end of file From d718c5af9952ce630bae1771dd0eb6839f6ad1f6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 4 Sep 2020 17:36:08 +0300 Subject: [PATCH 014/390] fixed --- src/Columns/ColumnNullable.cpp | 129 +++++++++++------- .../01457_order_by_nulls_first.reference | 76 +++++++++++ .../01457_order_by_nulls_first.sql | 72 +++++++++- 3 files changed, 230 insertions(+), 47 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 888410202f0..caebe28e510 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -329,73 +329,110 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi } } -void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const +void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - if (limit >= equal_range.back().second || limit >= size()) + if (limit >= equal_ranges.back().second || limit >= size()) limit = 0; - EqualRanges new_ranges, temp_ranges; + EqualRanges new_ranges; - for (const auto &[first, last] : equal_range) + const auto is_nulls_last = ((null_direction_hint > 0) != reverse); + + if (is_nulls_last) { - bool direction = ((null_direction_hint > 0) != reverse); /// Shift all NULL values to the end. - - size_t read_idx = first; - size_t write_idx = first; - while (read_idx < last && (isNullAt(res[read_idx])^direction)) + for (const auto & [first, last] : equal_ranges) { - ++read_idx; - ++write_idx; - } + /// Consider a half interval [first, last) + size_t read_idx = first; + size_t write_idx = first; + size_t end_idx = last; - ++read_idx; + if (!limit) + limit = end_idx; + else + limit = std::min(end_idx - first + 1, limit); - /// Invariants: - /// write_idx < read_idx - /// write_idx points to NULL - /// read_idx will be incremented to position of next not-NULL - /// there are range of NULLs between write_idx and read_idx - 1, - /// We are moving elements from end to begin of this range, - /// so range will "bubble" towards the end. - /// Relative order of NULL elements could be changed, - /// but relative order of non-NULLs is preserved. - - while (read_idx < last && write_idx < last) - { - if (isNullAt(res[read_idx])^direction) + while (read_idx < limit && !isNullAt(res[read_idx])) { - std::swap(res[read_idx], res[write_idx]); + ++read_idx; ++write_idx; } - ++read_idx; - } - if (write_idx - first > 1) - { - if (direction) - temp_ranges.emplace_back(first, write_idx); - else + ++read_idx; + + /// Invariants: + /// write_idx < read_idx + /// write_idx points to NULL + /// read_idx will be incremented to position of next not-NULL + /// there are range of NULLs between write_idx and read_idx - 1, + /// We are moving elements from end to begin of this range, + /// so range will "bubble" towards the end. + /// Relative order of NULL elements could be changed, + /// but relative order of non-NULLs is preserved. + + while (read_idx < end_idx && write_idx < limit) + { + if (!isNullAt(res[read_idx])) + { + std::swap(res[read_idx], res[write_idx]); + ++write_idx; + } + ++read_idx; + } + + /// We have a range [first, write_idx) of non-NULL values + if (first != write_idx) new_ranges.emplace_back(first, write_idx); - } - if (last - write_idx > 1) - { - if (direction) + /// We have a range [write_idx, list) of NULL values + if (write_idx != last) new_ranges.emplace_back(write_idx, last); - else - temp_ranges.emplace_back(write_idx, last); } } - while (!new_ranges.empty() && limit && limit <= new_ranges.back().first) - new_ranges.pop_back(); + else + { + for (const auto & [first, last] : equal_ranges) + { + /// Shift all NULL values to the beginning. - if (!temp_ranges.empty()) - getNestedColumn().updatePermutation(reverse, limit, null_direction_hint, res, temp_ranges); + ssize_t read_idx = last - 1; + ssize_t write_idx = last - 1; + ssize_t begin_idx = first; - equal_range.resize(temp_ranges.size() + new_ranges.size()); - std::merge(temp_ranges.begin(), temp_ranges.end(), new_ranges.begin(), new_ranges.end(), equal_range.begin()); + while (read_idx >= begin_idx && !isNullAt(res[read_idx])) + { + --read_idx; + --write_idx; + } + + --read_idx; + + while (read_idx >= begin_idx && write_idx >= begin_idx) + { + if (!isNullAt(res[read_idx])) + { + std::swap(res[read_idx], res[write_idx]); + --write_idx; + } + --read_idx; + } + + /// We have a range [write_idx+1, last) of non-NULL values + if (write_idx != static_cast(last)) + new_ranges.emplace_back(write_idx + 1, last); + + + /// We have a range [first, write_idx+1) of NULL values + if (static_cast(first) != write_idx) + new_ranges.emplace_back(first, write_idx + 1); + } + } + + getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); + + equal_ranges = std::move(new_ranges); } void ColumnNullable::gather(ColumnGathererStream & gatherer) diff --git a/tests/queries/0_stateless/01457_order_by_nulls_first.reference b/tests/queries/0_stateless/01457_order_by_nulls_first.reference index e69de29bb2d..355e58120fe 100644 --- a/tests/queries/0_stateless/01457_order_by_nulls_first.reference +++ b/tests/queries/0_stateless/01457_order_by_nulls_first.reference @@ -0,0 +1,76 @@ +\N 0 +\N 0 +\N 0 +\N 0 +--- DESC NULLS FIRST, ASC +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 1 +28 0 +0 0 +--- DESC NULLS LAST, ASC +28 0 +0 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 1 +--- ASC NULLS FIRST, ASC +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 1 +0 0 +28 0 +--- ASC NULLS LAST, ASC +0 0 +28 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +\N 1 +--- DESC NULLS FIRST, DESC +\N 1 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +28 0 +0 0 +--- DESC NULLS LAST, DESC +28 0 +0 0 +\N 1 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +--- ASC NULLS FIRST, DESC +\N 1 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 +0 0 +28 0 +--- ASC NULLS LAST, DESC +0 0 +28 0 +\N 1 +\N 0 +\N 0 +\N 0 +\N 0 +\N 0 diff --git a/tests/queries/0_stateless/01457_order_by_nulls_first.sql b/tests/queries/0_stateless/01457_order_by_nulls_first.sql index 7e391276d84..100c87fbead 100644 --- a/tests/queries/0_stateless/01457_order_by_nulls_first.sql +++ b/tests/queries/0_stateless/01457_order_by_nulls_first.sql @@ -13,7 +13,7 @@ FROM order_by_nulls_first order by diff desc NULLS FIRST, traf limit 1, 4; -select '---'; +select '--- DESC NULLS FIRST, ASC'; SELECT diff, @@ -23,4 +23,74 @@ ORDER BY diff DESC NULLS FIRST, traf ASC; +select '--- DESC NULLS LAST, ASC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff DESC NULLS LAST, + traf ASC; + +select '--- ASC NULLS FIRST, ASC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff ASC NULLS FIRST, + traf ASC; + +select '--- ASC NULLS LAST, ASC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff ASC NULLS LAST, + traf ASC; + +select '--- DESC NULLS FIRST, DESC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff DESC NULLS FIRST, + traf DESC; + +select '--- DESC NULLS LAST, DESC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff DESC NULLS LAST, + traf DESC; + +select '--- ASC NULLS FIRST, DESC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff ASC NULLS FIRST, + traf DESC; + +select '--- ASC NULLS LAST, DESC'; + +SELECT + diff, + traf +FROM order_by_nulls_first +ORDER BY + diff ASC NULLS LAST, + traf DESC; + drop table if exists order_by_nulls_first; \ No newline at end of file From f67a7b3a3d855c6e0d07a1bda10faaf414270327 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 4 Sep 2020 19:53:50 +0300 Subject: [PATCH 015/390] better --- src/Columns/ColumnNullable.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index caebe28e510..6033cdad53a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -334,7 +334,8 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (limit >= equal_ranges.back().second || limit >= size()) limit = 0; - EqualRanges new_ranges; + /// We will sort nested columns into `new_ranges` and call updatePermutation in next columns with `null_ranges`. + EqualRanges new_ranges, null_ranges; const auto is_nulls_last = ((null_direction_hint > 0) != reverse); @@ -388,7 +389,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// We have a range [write_idx, list) of NULL values if (write_idx != last) - new_ranges.emplace_back(write_idx, last); + null_ranges.emplace_back(write_idx, last); } } else @@ -426,13 +427,14 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// We have a range [first, write_idx+1) of NULL values if (static_cast(first) != write_idx) - new_ranges.emplace_back(first, write_idx + 1); + null_ranges.emplace_back(first, write_idx + 1); } } getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); equal_ranges = std::move(new_ranges); + equal_ranges.insert(equal_ranges.end(), null_ranges.begin(), null_ranges.end()); } void ColumnNullable::gather(ColumnGathererStream & gatherer) From 4fadb6c3ecb826807f4e1ec17419e7e6b36b4982 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 4 Sep 2020 21:05:06 +0300 Subject: [PATCH 016/390] fast test --- src/Columns/ColumnArray.cpp | 3 +++ src/Columns/ColumnDecimal.cpp | 3 +++ src/Columns/ColumnFixedString.cpp | 3 +++ src/Columns/ColumnLowCardinality.cpp | 3 +++ src/Columns/ColumnNullable.cpp | 10 +++++++++- src/Columns/ColumnString.cpp | 3 +++ src/Columns/ColumnTuple.cpp | 3 +++ src/Columns/ColumnUnique.h | 3 +++ src/Columns/ColumnVector.cpp | 3 +++ 9 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 58543d6a4dd..cd4aa57c18f 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -781,6 +781,9 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 6b4341a697e..6bab4228e9d 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -144,6 +144,9 @@ void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn: template void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= data.size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 95a477e54cf..c10caa37b28 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -170,6 +170,9 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 5c174d57b32..0613e5e2b71 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -331,6 +331,9 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= size() || limit >= equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 6033cdad53a..1846e066bed 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -331,6 +331,9 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { + if (equal_ranges.empty()) + return; + if (limit >= equal_ranges.back().second || limit >= size()) limit = 0; @@ -433,8 +436,13 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); + std::cout << "new_ranges " << new_ranges.size() << std::endl; + std::cout << "null_ranges " << null_ranges.size() << std::endl; + equal_ranges = std::move(new_ranges); - equal_ranges.insert(equal_ranges.end(), null_ranges.begin(), null_ranges.end()); + std::move(null_ranges.begin(), null_ranges.end(), std::back_inserter(equal_ranges)); + + std::cout << "end" << std::endl; } void ColumnNullable::gather(ColumnGathererStream & gatherer) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 6c84107caae..57795535a64 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -327,6 +327,9 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= size() || limit > equal_range.back().second) limit = 0; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 87e5e37db51..09c7472b22b 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -346,6 +346,9 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + for (const auto& column : columns) { column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index d0edf65edd8..59febe52112 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -389,6 +389,9 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, template void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + bool found_null_value_index = false; for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i) { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index b1b3ed4478a..416123af8f0 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -243,6 +243,9 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi template void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { + if (equal_range.empty()) + return; + if (limit >= data.size() || limit >= equal_range.back().second) limit = 0; From f757438e71d8e7a973d778fc0a5f0e54effc8130 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Fri, 4 Sep 2020 22:03:16 +0300 Subject: [PATCH 017/390] Update docs/en/operations/settings/settings.md Co-authored-by: BayoNet --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 11ab1247753..d4edc22a89b 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1901,7 +1901,7 @@ Default value: `120`. ## output_format_pretty_max_value_width {#output_format_pretty_max_value_width} -Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pretty) formats. If the value width exceeds the setting, the value is cut. +Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pretty) formats. If the value width exceeds the limit, the value is cut. Possible values: From e948327b441f3dd499537976509234f82e1bcd19 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Fri, 4 Sep 2020 23:30:25 +0300 Subject: [PATCH 018/390] Translated into Russian. --- docs/ru/operations/settings/settings.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 62c40c90540..b04f8f411c3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1643,4 +1643,58 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; - [Секции и настройки запроса CREATE TABLE](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) (настройка `merge_with_ttl_timeout`) - [Table TTL](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) +## output_format_pretty_max_value_width {#output_format_pretty_max_value_width} + +Ограничивает длину значения, выводимого в формате [Pretty](../../interfaces/formats.md#pretty). Если значение длиннее указанного количества символов, оно обрезается. + +Возможные значения: + +- Положительное целое число. +- 0 — значение обрезается полностью. + +Значение по умолчанию: `10000` символов. + +**Примеры** + +Запрос: + +```sql +SET output_format_pretty_max_value_width = 10; +SELECT range(number) FROM system.numbers LIMIT 10 FORMAT PrettyCompactNoEscapes; +``` +Результат: + +```text +┌─range(number)─┐ +│ [] │ +│ [0] │ +│ [0,1] │ +│ [0,1,2] │ +│ [0,1,2,3] │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +│ [0,1,2,3,4⋯ │ +└───────────────┘ +``` + +Запрос, где длина выводимого значения ограничена 0 символов: + +```sql +SET output_format_pretty_max_value_width = 0; +SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; +``` +Результат: + +```text +┌─range(number)─┐ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +│ ⋯ │ +└───────────────┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From a3671a15f61d4958b5184e97d4048bc454b816c5 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 7 Sep 2020 10:54:59 +0800 Subject: [PATCH 019/390] Add new feature: SHOW DATABASES LIKE --- .../InterpreterShowTablesQuery.cpp | 16 +++++++++++++++- src/Parsers/ASTShowTablesQuery.cpp | 14 ++++++++++++++ src/Parsers/ParserShowTablesQuery.cpp | 19 +++++++++++++++++++ .../01470_show_databases_like.reference | 1 + .../0_stateless/01470_show_databases_like.sql | 3 +++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01470_show_databases_like.reference create mode 100644 tests/queries/0_stateless/01470_show_databases_like.sql diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 4b0d4c21ad1..f8e387ef529 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -31,7 +31,21 @@ String InterpreterShowTablesQuery::getRewrittenQuery() /// SHOW DATABASES if (query.databases) - return "SELECT name FROM system.databases"; + { + std::stringstream rewritten_query; + rewritten_query << "SELECT name FROM system.databases"; + + if (!query.like.empty()) + { + rewritten_query << " WHERE name " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + } + + if (query.limit_length) + rewritten_query << " LIMIT " << query.limit_length; + + DUMP(rewritten_query.str()); + return rewritten_query.str(); + } /// SHOW CLUSTER/CLUSTERS if (query.clusters) diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index 5a284109cf2..ce44d2b56de 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -18,6 +18,20 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format if (databases) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW DATABASES" << (settings.hilite ? hilite_none : ""); + + if (!like.empty()) + settings.ostr + << (settings.hilite ? hilite_keyword : "") + << (not_like ? " NOT" : "") + << (case_insensitive_like ? " ILIKE " : " LIKE ") + << (settings.hilite ? hilite_none : "") + << std::quoted(like, '\''); + + if (limit_length) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(settings, state, frame); + } } else if (clusters) { diff --git a/src/Parsers/ParserShowTablesQuery.cpp b/src/Parsers/ParserShowTablesQuery.cpp index 66ecdf61c58..4586e10a8a3 100644 --- a/src/Parsers/ParserShowTablesQuery.cpp +++ b/src/Parsers/ParserShowTablesQuery.cpp @@ -46,6 +46,25 @@ bool ParserShowTablesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (s_databases.ignore(pos)) { query->databases = true; + + if (s_not.ignore(pos, expected)) + query->not_like = true; + + if (bool insensitive = s_ilike.ignore(pos, expected); insensitive || s_like.ignore(pos, expected)) + { + if (insensitive) + query->case_insensitive_like = true; + + if (!like_p.parse(pos, like, expected)) + return false; + } + else if (query->not_like) + return false; + if (s_limit.ignore(pos, expected)) + { + if (!exp_elem.parse(pos, query->limit_length, expected)) + return false; + } } else if (s_clusters.ignore(pos)) { diff --git a/tests/queries/0_stateless/01470_show_databases_like.reference b/tests/queries/0_stateless/01470_show_databases_like.reference new file mode 100644 index 00000000000..19b0eb327f5 --- /dev/null +++ b/tests/queries/0_stateless/01470_show_databases_like.reference @@ -0,0 +1 @@ +test_01470 diff --git a/tests/queries/0_stateless/01470_show_databases_like.sql b/tests/queries/0_stateless/01470_show_databases_like.sql new file mode 100644 index 00000000000..46ec8878105 --- /dev/null +++ b/tests/queries/0_stateless/01470_show_databases_like.sql @@ -0,0 +1,3 @@ +create database if not exists test_01470; +show databases like '%01470'; +drop database test_01470; From dc0e276bba24425c9355874e17a28874cfd7e336 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 7 Sep 2020 11:06:21 +0800 Subject: [PATCH 020/390] Add new feature: SHOW DATABASES LIKE --- src/Interpreters/InterpreterShowTablesQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index f8e387ef529..09c617e12ec 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -43,7 +43,6 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.limit_length) rewritten_query << " LIMIT " << query.limit_length; - DUMP(rewritten_query.str()); return rewritten_query.str(); } From b7e9d5e72dcc926d3e9ee3bd426354e93b7d2b5a Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 5 Sep 2020 13:46:51 +0800 Subject: [PATCH 021/390] ISSUES-4006 support decimal for MaterializedMySQL engine --- src/Core/MySQL/IMySQLReadPacket.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 143 +++++++++--------- .../MySQL/MaterializeMySQLSyncThread.cpp | 10 +- 3 files changed, 80 insertions(+), 75 deletions(-) diff --git a/src/Core/MySQL/IMySQLReadPacket.h b/src/Core/MySQL/IMySQLReadPacket.h index eab31889091..7484e7acc89 100644 --- a/src/Core/MySQL/IMySQLReadPacket.h +++ b/src/Core/MySQL/IMySQLReadPacket.h @@ -25,7 +25,7 @@ protected: virtual void readPayloadImpl(ReadBuffer & buf) = 0; }; -class LimitedReadPacket : public IMySQLReadPacket + class LimitedReadPacket : public IMySQLReadPacket { public: void readPayload(ReadBuffer & in, uint8_t & sequence_id) override; diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 6a102dca1f3..07bc4773882 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -467,87 +467,86 @@ namespace MySQLReplication } case MYSQL_TYPE_NEWDECIMAL: { - Int8 digits_per_integer = 9; - Int8 precision = meta >> 8; - Int8 decimals = meta & 0xff; - const char compressed_byte_map[] = {0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; - - Int8 integral = (precision - decimals); - UInt32 uncompressed_integers = integral / digits_per_integer; - UInt32 uncompressed_decimals = decimals / digits_per_integer; - UInt32 compressed_integers = integral - (uncompressed_integers * digits_per_integer); - UInt32 compressed_decimals = decimals - (uncompressed_decimals * digits_per_integer); - - String buff; - UInt32 bytes_to_read = uncompressed_integers * 4 + compressed_byte_map[compressed_integers] - + uncompressed_decimals * 4 + compressed_byte_map[compressed_decimals]; - buff.resize(bytes_to_read); - payload.readStrict(reinterpret_cast(buff.data()), bytes_to_read); - - String format; - format.resize(0); - - bool is_negative = ((buff[0] & 0x80) == 0); - if (is_negative) + const auto & dispatch = [](const size_t & precision, const size_t & scale, const auto & function) -> Field { - format += "-"; - } - buff[0] ^= 0x80; + if (precision <= DecimalUtils::maxPrecision()) + return Field(function(precision, scale, Decimal32())); + else if (precision <= DecimalUtils::maxPrecision()) + return Field(function(precision, scale, Decimal64())); + else if (precision <= DecimalUtils::maxPrecision()) + return Field(function(precision, scale, Decimal128())); - ReadBufferFromString reader(buff); - /// Compressed part. - if (compressed_integers != 0) - { - Int64 val = 0; - UInt8 to_read = compressed_byte_map[compressed_integers]; - readBigEndianStrict(reader, reinterpret_cast(&val), to_read); - format += std::to_string(val); - } + return Field(function(precision, scale, Decimal256())); + }; - for (auto k = 0U; k < uncompressed_integers; k++) + const auto & read_decimal = [&](const size_t & precision, const size_t & scale, auto decimal) { - UInt32 val = 0; - readBigEndianStrict(reader, reinterpret_cast(&val), 4); - format += std::to_string(val); - } - format += "."; - for (auto k = 0U; k < uncompressed_decimals; k++) - { - UInt32 val = 0; - reader.readStrict(reinterpret_cast(&val), 4); - format += std::to_string(val); - } + using DecimalType = decltype(decimal); + static constexpr size_t digits_per_integer = 9; + static const size_t compressed_byte_map[] = {0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; + + DecimalType res(0); + bool is_negative = (*payload.position() & 0x80) == 0; + *payload.position() ^= 0x80; - /// Compressed part. - if (compressed_decimals != 0) - { - Int64 val = 0; - String compressed_buff; - UInt8 to_read = compressed_byte_map[compressed_decimals]; - switch (to_read) { - case 1: { - reader.readStrict(reinterpret_cast(&val), 1); - break; + size_t integral = (precision - scale); + size_t uncompressed_integers = integral / digits_per_integer; + size_t compressed_integers = integral - (uncompressed_integers * digits_per_integer); + + /// Compressed part. + if (compressed_integers != 0) + { + Int64 val = 0; + size_t to_read = compressed_byte_map[compressed_integers]; + readBigEndianStrict(payload, reinterpret_cast(&val), to_read); + res += val; } - case 2: { - readBigEndianStrict(reader, reinterpret_cast(&val), 2); - break; + + for (auto k = 0U; k < uncompressed_integers; k++) + { + UInt32 val = 0; + readBigEndianStrict(payload, reinterpret_cast(&val), 4); + res *= intExp10OfSize(k ? digits_per_integer : std::max(size_t(1), compressed_integers)); + res += val; } - case 3: { - readBigEndianStrict(reader, reinterpret_cast(&val), 3); - break; - } - case 4: { - readBigEndianStrict(reader, reinterpret_cast(&val), 4); - break; - } - default: - break; } - format += std::to_string(val); - } - row.push_back(Field{String{format}}); + + { + size_t uncompressed_decimals = scale / digits_per_integer; + size_t compressed_decimals = scale - (uncompressed_decimals * digits_per_integer); + + for (auto k = 0U; k < uncompressed_decimals; k++) + { + UInt32 val = 0; + payload.readStrict(reinterpret_cast(&val), 4); + res *= intExp10OfSize(digits_per_integer); + res += val; + } + + /// Compressed part. + if (compressed_decimals != 0) + { + Int64 val = 0; + String compressed_buff; + size_t to_read = compressed_byte_map[compressed_decimals]; + + if (to_read) + { + payload.readStrict(reinterpret_cast(&val), to_read); + res *= intExp10OfSize(compressed_decimals); + res += val; + } + } + } + + if (is_negative) + res *= -1; + + return res; + }; + + row.push_back(dispatch((meta >> 8) & 0xFF, meta & 0xFF, read_decimal)); break; } case MYSQL_TYPE_VARCHAR: diff --git a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp index 7ded256dd86..3a26e25d08f 100644 --- a/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSyncThread.cpp @@ -452,8 +452,14 @@ static void writeFieldsToColumn( write_data_to_column(casted_float32_column, Float64(), Float32()); else if (ColumnFloat64 * casted_float64_column = typeid_cast(&column_to)) write_data_to_column(casted_float64_column, Float64(), Float64()); - else if (ColumnDecimal * casted_date_time_64_column = typeid_cast *>(&column_to)) - write_data_to_column(casted_date_time_64_column, DateTime64(), DateTime64()); + else if (ColumnDecimal * casted_decimal_32_column = typeid_cast *>(&column_to)) + write_data_to_column(casted_decimal_32_column, Decimal32(), Decimal32()); + else if (ColumnDecimal * casted_decimal_64_column = typeid_cast *>(&column_to)) + write_data_to_column(casted_decimal_64_column, Decimal64(), Decimal64()); + else if (ColumnDecimal * casted_decimal_128_column = typeid_cast *>(&column_to)) + write_data_to_column(casted_decimal_128_column, Decimal128(), Decimal128()); + else if (ColumnDecimal * casted_decimal_256_column = typeid_cast *>(&column_to)) + write_data_to_column(casted_decimal_256_column, Decimal256(), Decimal256()); else if (ColumnInt32 * casted_int32_column = typeid_cast(&column_to)) { for (size_t index = 0; index < rows_data.size(); ++index) From c09d86e5e460e466bf25ef382cdf0d499112eff4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 7 Sep 2020 16:15:04 +0300 Subject: [PATCH 022/390] another fixes --- src/Columns/ColumnNullable.cpp | 35 ++++++++++++++++++++++++++-------- src/Columns/ColumnVector.cpp | 6 ++++++ src/Interpreters/sortBlock.cpp | 32 +++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 1846e066bed..e5771ceed5c 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -331,6 +331,12 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { +// std::cout << "ColumnNullable" << std::endl; +// +// std::cout << "equal_ranges " << equal_ranges.size() << std::endl; +// for (auto [first, last] : equal_ranges) +// std::cout << "first " << first << " last " << last << std::endl; + if (equal_ranges.empty()) return; @@ -344,20 +350,24 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (is_nulls_last) { +// std::cout << "NULL LAST" << std::endl; /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { +// std::cout << "current range " << first << ' ' << last << std::endl; /// Consider a half interval [first, last) size_t read_idx = first; size_t write_idx = first; size_t end_idx = last; if (!limit) - limit = end_idx; + limit = end_idx - read_idx; else - limit = std::min(end_idx - first + 1, limit); + limit = std::min(end_idx - read_idx, limit); - while (read_idx < limit && !isNullAt(res[read_idx])) + /// We simply check the limit not to do extra work. + /// Since interval begins from `first`, not from zero, we add `first` to the right side of the inequality. + while (read_idx < first + limit && !isNullAt(res[read_idx])) { ++read_idx; ++write_idx; @@ -375,7 +385,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Relative order of NULL elements could be changed, /// but relative order of non-NULLs is preserved. - while (read_idx < end_idx && write_idx < limit) + while (read_idx < end_idx && write_idx < first + limit) { if (!isNullAt(res[read_idx])) { @@ -397,6 +407,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } else { +// std::cout << "NULLS FIRST" << std::endl; for (const auto & [first, last] : equal_ranges) { /// Shift all NULL values to the beginning. @@ -436,13 +447,21 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); - std::cout << "new_ranges " << new_ranges.size() << std::endl; - std::cout << "null_ranges " << null_ranges.size() << std::endl; - +// std::cout << "new_ranges " << new_ranges.size() << std::endl; +// for (auto [first, last] : new_ranges) +// std::cout << "first " << first << " last " << last << std::endl; +// std::cout << "null_ranges " << null_ranges.size() << std::endl; +// for (auto [first, last] : null_ranges) +// std::cout << "first " << first << " last " << last << std::endl; +// equal_ranges = std::move(new_ranges); std::move(null_ranges.begin(), null_ranges.end(), std::back_inserter(equal_ranges)); - std::cout << "end" << std::endl; +// std::cout << "equal_ranges_final " << equal_ranges.size() << std::endl; +// for (auto [first, last] : equal_ranges) +// std::cout << "first " << first << " last " << last << std::endl; + +// std::cout << "end" << std::endl; } void ColumnNullable::gather(ColumnGathererStream & gatherer) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 416123af8f0..b42d9409a2a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -243,6 +243,12 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi template void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { +// std::cout << "ColumnVector" << std::endl; +// +// std::cout << "equal_ranges " << equal_range.size() << std::endl; +// for (auto [first, last] : equal_range) +// std::cout << "first " << first << " last " << last << std::endl; + if (equal_range.empty()) return; diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index cb3c36e5356..1a8b80cbadb 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -104,6 +104,19 @@ struct PartialSortingLessWithCollation void sortBlock(Block & block, const SortDescription & description, UInt64 limit) { +// std::cout << block.dumpStructure() << std::endl; +// +// for (const auto & column : block.getColumnsWithTypeAndName()) +// { +// std::cout << column.name << " \t\t"; +// auto column_size = column.column->size(); +// for (size_t i = 0; i < column_size; ++i) +// { +// std::cout << toString(column.column->operator[](i)) << ", \t"; +// } +// std::cout << std::endl; +// } + if (!block) return; @@ -181,6 +194,8 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto & column : columns_with_sort_desc) { +// std::cout << "need collation" << std::endl; +// std::cout << column.column->dumpStructure() << std::endl; while (!ranges.empty() && limit && limit <= ranges.back().first) ranges.pop_back(); @@ -210,6 +225,9 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto & column : columns_with_sort_desc) { +// std::cout << "no need collation" << std::endl; +// std::cout << column.column->dumpStructure() << std::endl; + while (!ranges.empty() && limit && limit <= ranges.back().first) { ranges.pop_back(); @@ -229,6 +247,20 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit); } } +// +// std::cout << "final block" << std::endl; +// std::cout << block.dumpStructure() << std::endl; +// +// for (const auto & column : block.getColumnsWithTypeAndName()) +// { +// std::cout << column.name << " \t\t"; +// auto column_size = column.column->size(); +// for (size_t i = 0; i < column_size; ++i) +// { +// std::cout << toString(column.column->operator[](i)) << ", \t"; +// } +// std::cout << std::endl; +// } } From 8793281e3e6d6cde788e044af168893d11efe146 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 7 Sep 2020 17:02:59 +0300 Subject: [PATCH 023/390] remove cout --- src/Columns/ColumnNullable.cpp | 22 ---------------------- src/Columns/ColumnVector.cpp | 6 ------ src/Interpreters/sortBlock.cpp | 32 -------------------------------- 3 files changed, 60 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index e5771ceed5c..12934b9420b 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -331,12 +331,6 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { -// std::cout << "ColumnNullable" << std::endl; -// -// std::cout << "equal_ranges " << equal_ranges.size() << std::endl; -// for (auto [first, last] : equal_ranges) -// std::cout << "first " << first << " last " << last << std::endl; - if (equal_ranges.empty()) return; @@ -350,11 +344,9 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (is_nulls_last) { -// std::cout << "NULL LAST" << std::endl; /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { -// std::cout << "current range " << first << ' ' << last << std::endl; /// Consider a half interval [first, last) size_t read_idx = first; size_t write_idx = first; @@ -407,7 +399,6 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } else { -// std::cout << "NULLS FIRST" << std::endl; for (const auto & [first, last] : equal_ranges) { /// Shift all NULL values to the beginning. @@ -447,21 +438,8 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); -// std::cout << "new_ranges " << new_ranges.size() << std::endl; -// for (auto [first, last] : new_ranges) -// std::cout << "first " << first << " last " << last << std::endl; -// std::cout << "null_ranges " << null_ranges.size() << std::endl; -// for (auto [first, last] : null_ranges) -// std::cout << "first " << first << " last " << last << std::endl; -// equal_ranges = std::move(new_ranges); std::move(null_ranges.begin(), null_ranges.end(), std::back_inserter(equal_ranges)); - -// std::cout << "equal_ranges_final " << equal_ranges.size() << std::endl; -// for (auto [first, last] : equal_ranges) -// std::cout << "first " << first << " last " << last << std::endl; - -// std::cout << "end" << std::endl; } void ColumnNullable::gather(ColumnGathererStream & gatherer) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index b42d9409a2a..416123af8f0 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -243,12 +243,6 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi template void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { -// std::cout << "ColumnVector" << std::endl; -// -// std::cout << "equal_ranges " << equal_range.size() << std::endl; -// for (auto [first, last] : equal_range) -// std::cout << "first " << first << " last " << last << std::endl; - if (equal_range.empty()) return; diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 1a8b80cbadb..cb3c36e5356 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -104,19 +104,6 @@ struct PartialSortingLessWithCollation void sortBlock(Block & block, const SortDescription & description, UInt64 limit) { -// std::cout << block.dumpStructure() << std::endl; -// -// for (const auto & column : block.getColumnsWithTypeAndName()) -// { -// std::cout << column.name << " \t\t"; -// auto column_size = column.column->size(); -// for (size_t i = 0; i < column_size; ++i) -// { -// std::cout << toString(column.column->operator[](i)) << ", \t"; -// } -// std::cout << std::endl; -// } - if (!block) return; @@ -194,8 +181,6 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto & column : columns_with_sort_desc) { -// std::cout << "need collation" << std::endl; -// std::cout << column.column->dumpStructure() << std::endl; while (!ranges.empty() && limit && limit <= ranges.back().first) ranges.pop_back(); @@ -225,9 +210,6 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) ranges.emplace_back(0, perm.size()); for (const auto & column : columns_with_sort_desc) { -// std::cout << "no need collation" << std::endl; -// std::cout << column.column->dumpStructure() << std::endl; - while (!ranges.empty() && limit && limit <= ranges.back().first) { ranges.pop_back(); @@ -247,20 +229,6 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit); } } -// -// std::cout << "final block" << std::endl; -// std::cout << block.dumpStructure() << std::endl; -// -// for (const auto & column : block.getColumnsWithTypeAndName()) -// { -// std::cout << column.name << " \t\t"; -// auto column_size = column.column->size(); -// for (size_t i = 0; i < column_size; ++i) -// { -// std::cout << toString(column.column->operator[](i)) << ", \t"; -// } -// std::cout << std::endl; -// } } From 46c84b054e8c1a041a5dd58df249f22ca25122d1 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 8 Sep 2020 01:28:20 +0800 Subject: [PATCH 024/390] ISSUES-4006 fix negative decimal number --- src/Core/MySQL/IMySQLReadPacket.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 31 +++++++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Core/MySQL/IMySQLReadPacket.h b/src/Core/MySQL/IMySQLReadPacket.h index 7484e7acc89..eab31889091 100644 --- a/src/Core/MySQL/IMySQLReadPacket.h +++ b/src/Core/MySQL/IMySQLReadPacket.h @@ -25,7 +25,7 @@ protected: virtual void readPayloadImpl(ReadBuffer & buf) = 0; }; - class LimitedReadPacket : public IMySQLReadPacket +class LimitedReadPacket : public IMySQLReadPacket { public: void readPayload(ReadBuffer & in, uint8_t & sequence_id) override; diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 07bc4773882..50bbe9aaaf7 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -483,10 +483,16 @@ namespace MySQLReplication { using DecimalType = decltype(decimal); static constexpr size_t digits_per_integer = 9; - static const size_t compressed_byte_map[] = {0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; + static const size_t compressed_bytes_map[] = {0, 1, 1, 2, 2, 3, 3, 4, 4, 4}; + static const size_t compressed_integer_align_numbers[] = { + 0x0, 0xFF, 0xFF, 0xFFFF, 0xFFFF, 0xFFFFFF, 0xFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; + UInt32 mask = 0; DecimalType res(0); - bool is_negative = (*payload.position() & 0x80) == 0; + + if ((*payload.position() & 0x80) == 0) + mask = UInt32(-1); + *payload.position() ^= 0x80; { @@ -497,18 +503,18 @@ namespace MySQLReplication /// Compressed part. if (compressed_integers != 0) { - Int64 val = 0; - size_t to_read = compressed_byte_map[compressed_integers]; + UInt32 val = 0; + size_t to_read = compressed_bytes_map[compressed_integers]; readBigEndianStrict(payload, reinterpret_cast(&val), to_read); - res += val; + res += (val ^ (mask & compressed_integer_align_numbers[compressed_integers])); } for (auto k = 0U; k < uncompressed_integers; k++) { UInt32 val = 0; readBigEndianStrict(payload, reinterpret_cast(&val), 4); - res *= intExp10OfSize(k ? digits_per_integer : std::max(size_t(1), compressed_integers)); - res += val; + res *= intExp10OfSize(digits_per_integer); + res += (val ^ mask); } } @@ -521,26 +527,25 @@ namespace MySQLReplication UInt32 val = 0; payload.readStrict(reinterpret_cast(&val), 4); res *= intExp10OfSize(digits_per_integer); - res += val; + res += (val ^ mask); } /// Compressed part. if (compressed_decimals != 0) { - Int64 val = 0; - String compressed_buff; - size_t to_read = compressed_byte_map[compressed_decimals]; + UInt32 val = 0; + size_t to_read = compressed_bytes_map[compressed_decimals]; if (to_read) { payload.readStrict(reinterpret_cast(&val), to_read); res *= intExp10OfSize(compressed_decimals); - res += val; + res += (val ^ (mask & compressed_integer_align_numbers[compressed_decimals])); } } } - if (is_negative) + if (mask != 0) res *= -1; return res; From 564dfac737802b2bb8e39f98eb8914587f7bc97b Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 8 Sep 2020 02:36:07 +0800 Subject: [PATCH 025/390] ISSUES-4006 fix decimal type decimal number part --- src/Core/MySQL/MySQLReplication.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 50bbe9aaaf7..a277c183653 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -525,7 +525,7 @@ namespace MySQLReplication for (auto k = 0U; k < uncompressed_decimals; k++) { UInt32 val = 0; - payload.readStrict(reinterpret_cast(&val), 4); + readBigEndianStrict(payload, reinterpret_cast(&val), 4); res *= intExp10OfSize(digits_per_integer); res += (val ^ mask); } @@ -538,7 +538,7 @@ namespace MySQLReplication if (to_read) { - payload.readStrict(reinterpret_cast(&val), to_read); + readBigEndianStrict(payload, reinterpret_cast(&val), to_read); res *= intExp10OfSize(compressed_decimals); res += (val ^ (mask & compressed_integer_align_numbers[compressed_decimals])); } From 05bd0b7c28672aefbce78297a0e2f354a3a7ac4a Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 8 Sep 2020 02:36:28 +0800 Subject: [PATCH 026/390] ISSUES-4006 add integration test --- .../materialize_with_ddl.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 18695f40e53..2bddb7f7c84 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -95,6 +95,26 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam mysql_node.query("DROP DATABASE test_database") +def materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, mysql_node, service_name): + mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + mysql_node.query("CREATE TABLE test_database.test_table_1 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), " + "_decimal DECIMAL(65, 30)) ENGINE = InnoDB;") + mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + ('9' * 35) + "." + ('9' * 30) + ")") + mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + ('0' * 29) + "1)") + mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + ('9' * 35) + "." + ('9' * 30) + ")") + mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", + "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" + "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" + "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" + "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.999\t-0." + ('0' * 29) + "1\n") + clickhouse_node.query("DROP DATABASE test_database") + mysql_node.query("DROP DATABASE test_database") + + + def drop_table_with_materialize_mysql_database(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") mysql_node.query("CREATE TABLE test_database.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") From a150b1345bd669dadb0e37e9c6c6a438eebc1b7e Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 8 Sep 2020 11:29:18 +0800 Subject: [PATCH 027/390] ISSUES-4006 add integration test --- tests/integration/test_materialize_mysql_database/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index bfda4e7e840..46ade687b57 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -89,10 +89,13 @@ def started_mysql_8_0(): def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7): materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql5_7") + materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql5_7") def test_materialize_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0") + materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_8_0, "mysql8_0") + def test_materialize_database_ddl_with_mysql_5_7(started_cluster, started_mysql_5_7): try: From 77b214f7ab18017833c9bae7430319f47864341a Mon Sep 17 00:00:00 2001 From: Gao Qiang <30835199+dreamerfable@users.noreply.github.com> Date: Tue, 8 Sep 2020 22:59:19 +0800 Subject: [PATCH 028/390] Update custom-partitioning-key.md --- .../custom-partitioning-key.md | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/zh/engines/table-engines/mergetree-family/custom-partitioning-key.md index d7653ca05d6..cf3ac76c8ce 100644 --- a/docs/zh/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/zh/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -2,9 +2,9 @@ [MergeTree](mergetree.md) 系列的表（包括 [可复制表](replication.md) ）可以使用分区。基于 MergeTree 表的 [物化视图](../special/materializedview.md#materializedview) 也支持分区。 -一个分区是指按指定规则逻辑组合一起的表的记录集。可以按任意标准进行分区，如按月，按日或按事件类型。为了减少需要操作的数据，每个分区都是分开存储的。访问数据时，ClickHouse 尽量使用这些分区的最小子集。 +分区是在一个表中通过指定的规则划分而成的逻辑数据集。可以按任意标准进行分区，如按月，按日或按事件类型。为了减少需要操作的数据，每个分区都是分开存储的。访问数据时，ClickHouse 尽量使用这些分区的最小子集。 -分区是在 [建表](mergetree.md#table_engine-mergetree-creating-a-table) 的 `PARTITION BY expr` 子句中指定。分区键可以是关于列的任何表达式。例如，指定按月分区，表达式为 `toYYYYMM(date_column)`： +分区是在 [建表](mergetree.md#table_engine-mergetree-creating-a-table) 时通过 `PARTITION BY expr` 子句指定的。分区键可以是表中列的任意表达式。例如，指定按月分区，表达式为 `toYYYYMM(date_column)`： ``` sql CREATE TABLE visits @@ -30,10 +30,10 @@ ORDER BY (CounterID, StartDate, intHash32(UserID)); 新数据插入到表中时，这些数据会存储为按主键排序的新片段（块）。插入后 10-15 分钟，同一分区的各个片段会合并为一整个片段。 -!!! attention "注意" - 那些有相同分区表达式值的数据片段才会合并。这意味着 **你不应该用太精细的分区方案**（超过一千个分区）。否则，会因为文件系统中的文件数量和需要找开的文件描述符过多，导致 `SELECT` 查询效率不佳。 +!!! info "注意" + 那些有相同分区表达式值的数据片段才会合并。这意味着 **你不应该用太精细的分区方案**（超过一千个分区）。否则，会因为文件系统中的文件数量过多和需要打开的文件描述符过多，导致 `SELECT` 查询效率不佳。 -可以通过 [系统。零件](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#system_tables-parts) 表查看表片段和分区信息。例如，假设我们有一个 `visits` 表，按月分区。对 `system.parts` 表执行 `SELECT`： +可以通过 [system.parts](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#system_tables-parts) 表查看表片段和分区信息。例如，假设我们有一个 `visits` 表，按月分区。对 `system.parts` 表执行 `SELECT`： ``` sql SELECT @@ -44,55 +44,59 @@ FROM system.parts WHERE table = 'visits' ``` - ┌─partition─┬─name───────────┬─active─┐ - │ 201901 │ 201901_1_3_1 │ 0 │ - │ 201901 │ 201901_1_9_2 │ 1 │ - │ 201901 │ 201901_8_8_0 │ 0 │ - │ 201901 │ 201901_9_9_0 │ 0 │ - │ 201902 │ 201902_4_6_1 │ 1 │ - │ 201902 │ 201902_10_10_0 │ 1 │ - │ 201902 │ 201902_11_11_0 │ 1 │ - └───────────┴────────────────┴────────┘ +``` text +┌─partition─┬─name───────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 1 │ +│ 201902 │ 201902_11_11_0 │ 1 │ +└───────────┴────────────────┴────────┘ +``` `partition` 列存储分区的名称。此示例中有两个分区：`201901` 和 `201902`。在 [ALTER … PARTITION](#alter_manipulations-with-partitions) 语句中你可以使用该列值来指定分区名称。 `name` 列为分区中数据片段的名称。在 [ALTER ATTACH PART](#alter_attach-partition) 语句中你可以使用此列值中来指定片段名称。 -这里我们拆解下第一部分的名称：`201901_1_3_1`： +这里我们拆解下第一个数据片段的名称：`201901_1_3_1`： - `201901` 是分区名称。 - `1` 是数据块的最小编号。 - `3` 是数据块的最大编号。 - `1` 是块级别（即在由块组成的合并树中，该块在树中的深度）。 -!!! attention "注意" +!!! info "注意" 旧类型表的片段名称为：`20190117_20190123_2_2_0`（最小日期 - 最大日期 - 最小块编号 - 最大块编号 - 块级别）。 -`active` 列为片段状态。`1` 激活状态；`0` 非激活状态。非激活片段是那些在合并到较大片段之后剩余的源数据片段。损坏的数据片段也表示为非活动状态。 +`active` 列为片段状态。`1` 代表激活状态；`0` 代表非激活状态。非激活片段是那些在合并到较大片段之后剩余的源数据片段。损坏的数据片段也表示为非活动状态。 -正如在示例中所看到的，同一分区中有几个独立的片段（例如，`201901_1_3_1`和`201901_1_9_2`）。这意味着这些片段尚未合并。ClickHouse 大约在插入后15分钟定期报告合并操作，合并插入的数据片段。此外，你也可以使用 [OPTIMIZE](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#misc_operations-optimize) 语句直接执行合并。例： +正如在示例中所看到的，同一分区中有几个独立的片段（例如，`201901_1_3_1`和`201901_1_9_2`）。这意味着这些片段尚未合并。ClickHouse 会定期的对插入的数据片段进行合并，大约是在插入后15分钟左右。此外，你也可以使用 [OPTIMIZE](../../../sql-reference/statements/misc.md#misc_operations-optimize) 语句发起一个计划外的合并。例如： ``` sql OPTIMIZE TABLE visits PARTITION 201902; ``` - ┌─partition─┬─name───────────┬─active─┐ - │ 201901 │ 201901_1_3_1 │ 0 │ - │ 201901 │ 201901_1_9_2 │ 1 │ - │ 201901 │ 201901_8_8_0 │ 0 │ - │ 201901 │ 201901_9_9_0 │ 0 │ - │ 201902 │ 201902_4_6_1 │ 0 │ - │ 201902 │ 201902_4_11_2 │ 1 │ - │ 201902 │ 201902_10_10_0 │ 0 │ - │ 201902 │ 201902_11_11_0 │ 0 │ - └───────────┴────────────────┴────────┘ +``` +┌─partition─┬─name───────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 0 │ +│ 201902 │ 201902_4_11_2 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 0 │ +│ 201902 │ 201902_11_11_0 │ 0 │ +└───────────┴────────────────┴────────┘ +``` -非激活片段会在合并后的10分钟左右删除。 +非激活片段会在合并后的10分钟左右被删除。 查看片段和分区信息的另一种方法是进入表的目录：`/var/lib/clickhouse/data///`。例如： ``` bash -dev:/var/lib/clickhouse/data/default/visits$ ls -l +/var/lib/clickhouse/data/default/visits$ ls -l total 40 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 201901_1_3_1 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2 @@ -105,12 +109,12 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ``` -文件夹 ‘201901\_1\_1\_0’，‘201901\_1\_7\_1’ 等是片段的目录。每个片段都与一个对应的分区相关，并且只包含这个月的数据（本例中的表按月分区）。 +‘201901\_1\_1\_0’，‘201901\_1\_7\_1’ 等文件夹是数据片段的目录。每个片段都与一个对应的分区相关，并且只包含这个月的数据（本例中的表按月分区）。 -`detached` 目录存放着使用 [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) 语句从表中分离的片段。损坏的片段也会移到该目录，而不是删除。服务器不使用`detached`目录中的片段。可以随时添加，删除或修改此目录中的数据 – 在运行 [ATTACH](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#alter_attach-partition) 语句前，服务器不会感知到。 +`detached` 目录存放着使用 [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition) 语句从表中卸载的片段。损坏的片段不会被删除而是也会移到该目录下。服务器不会去使用`detached`目录中的数据片段。因此你可以随时添加，删除或修改此目录中的数据 – 在运行 [ATTACH](../../../sql-reference/statements/alter.md#alter_attach-partition) 语句前，服务器不会感知到。 注意，在操作服务器时，你不能手动更改文件系统上的片段集或其数据，因为服务器不会感知到这些修改。对于非复制表，可以在服务器停止时执行这些操作，但不建议这样做。对于复制表，在任何情况下都不要更改片段文件。 -ClickHouse 支持对分区执行这些操作：删除分区，从一个表复制到另一个表，或创建备份。了解分区的所有操作，请参阅 [分区和片段的操作](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#alter_manipulations-with-partitions) 一节。 +ClickHouse 支持对分区执行这些操作：删除分区，将分区从一个表复制到另一个表，或创建备份。了解分区的所有操作，请参阅 [分区和片段的操作](../../../sql-reference/statements/alter.md#alter_manipulations-with-partitions) 一节。 [来源文章](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) From c4f0465a7c2c7732f5977f05c6cb8f60de554509 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 8 Sep 2020 18:54:30 +0300 Subject: [PATCH 029/390] limit --- src/Columns/ColumnNullable.cpp | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 12934b9420b..27c126a979e 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -334,8 +334,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (equal_ranges.empty()) return; - if (limit >= equal_ranges.back().second || limit >= size()) - limit = 0; + std::cout << "limit " << limit << std::endl; /// We will sort nested columns into `new_ranges` and call updatePermutation in next columns with `null_ranges`. EqualRanges new_ranges, null_ranges; @@ -347,19 +346,22 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { + /// Current interval is righter than limit. + if (first > limit) + break; + /// Consider a half interval [first, last) size_t read_idx = first; size_t write_idx = first; size_t end_idx = last; - if (!limit) - limit = end_idx - read_idx; - else - limit = std::min(end_idx - read_idx, limit); + size_t current_limit = end_idx; + if (limit && limit >= read_idx && limit <= end_idx) + current_limit = limit; /// We simply check the limit not to do extra work. /// Since interval begins from `first`, not from zero, we add `first` to the right side of the inequality. - while (read_idx < first + limit && !isNullAt(res[read_idx])) + while (read_idx < current_limit && !isNullAt(res[read_idx])) { ++read_idx; ++write_idx; @@ -377,7 +379,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Relative order of NULL elements could be changed, /// but relative order of non-NULLs is preserved. - while (read_idx < end_idx && write_idx < first + limit) + while (read_idx < end_idx && write_idx < current_limit) { if (!isNullAt(res[read_idx])) { @@ -391,7 +393,6 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (first != write_idx) new_ranges.emplace_back(first, write_idx); - /// We have a range [write_idx, list) of NULL values if (write_idx != last) null_ranges.emplace_back(write_idx, last); @@ -399,9 +400,12 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } else { + /// Shift all NULL values to the beginning. for (const auto & [first, last] : equal_ranges) { - /// Shift all NULL values to the beginning. + /// Current interval is righter than limit. + if (first > limit) + break; ssize_t read_idx = last - 1; ssize_t write_idx = last - 1; @@ -429,13 +433,20 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (write_idx != static_cast(last)) new_ranges.emplace_back(write_idx + 1, last); - /// We have a range [first, write_idx+1) of NULL values if (static_cast(first) != write_idx) null_ranges.emplace_back(first, write_idx + 1); } } + std::cout << "New Ranges " << std::endl; + for (auto [first, last] : new_ranges ) + std::cout << "first " << first << " last " << last << std::endl; + + std::cout << "Null Ranges " << std::endl; + for (auto [first, last] : null_ranges) + std::cout << "first " << first << " last " << last << std::endl; + getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); equal_ranges = std::move(new_ranges); From 614e01b0e46de3f31891b83466f9d784bd19c7bb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 8 Sep 2020 23:12:55 +0300 Subject: [PATCH 030/390] better --- src/Columns/ColumnNullable.cpp | 16 +++------------- src/Columns/ColumnVector.cpp | 10 +++++++++- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 27c126a979e..0b1c306092a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -334,8 +334,6 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire if (equal_ranges.empty()) return; - std::cout << "limit " << limit << std::endl; - /// We will sort nested columns into `new_ranges` and call updatePermutation in next columns with `null_ranges`. EqualRanges new_ranges, null_ranges; @@ -347,7 +345,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire for (const auto & [first, last] : equal_ranges) { /// Current interval is righter than limit. - if (first > limit) + if (limit && first > limit) break; /// Consider a half interval [first, last) @@ -404,7 +402,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire for (const auto & [first, last] : equal_ranges) { /// Current interval is righter than limit. - if (first > limit) + if (limit && first > limit) break; ssize_t read_idx = last - 1; @@ -439,15 +437,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire } } - std::cout << "New Ranges " << std::endl; - for (auto [first, last] : new_ranges ) - std::cout << "first " << first << " last " << last << std::endl; - - std::cout << "Null Ranges " << std::endl; - for (auto [first, last] : null_ranges) - std::cout << "first " << first << " last " << last << std::endl; - - getNestedColumn().updatePermutation(reverse, 0, null_direction_hint, res, new_ranges); + getNestedColumn().updatePermutation(reverse, limit, null_direction_hint, res, new_ranges); equal_ranges = std::move(new_ranges); std::move(null_ranges.begin(), null_ranges.end(), std::back_inserter(equal_ranges)); diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 416123af8f0..d950c03d49b 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -18,6 +18,8 @@ #include #include +#include + #if !defined(ARCADIA_BUILD) # include # if USE_OPENCL @@ -250,6 +252,7 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire limit = 0; EqualRanges new_ranges; + SCOPE_EXIT({equal_range = std::move(new_ranges);}); for (size_t i = 0; i < equal_range.size() - bool(limit); ++i) { @@ -278,6 +281,12 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire if (limit) { const auto & [first, last] = equal_range.back(); + + if (limit < first || limit >= last) + return; + + /// Since then, we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint)); else @@ -310,7 +319,6 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire new_ranges.emplace_back(new_first, new_last); } } - equal_range = std::move(new_ranges); } template From 62ef728b2f20fed99b43afabeeca451807f4879d Mon Sep 17 00:00:00 2001 From: Winter Zhang Date: Wed, 9 Sep 2020 10:52:14 +0800 Subject: [PATCH 031/390] ISSUES-4006 try fix test failure --- .../test_materialize_mysql_database/materialize_with_ddl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 00b6a87cb37..fa31c43e6c1 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -110,7 +110,7 @@ def materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, mysql_ "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" - "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.999\t-0." + ('0' * 29) + "1\n") + "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + ('0' * 29) + "1\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") From 406f384a45bce0a728e4e8cad06fd34b1577154f Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Wed, 9 Sep 2020 11:24:47 +0800 Subject: [PATCH 032/390] support ILIKE, and separate like,limit function. --- .../InterpreterShowTablesQuery.cpp | 18 ++++- src/Parsers/ASTShowTablesQuery.cpp | 66 ++++++++----------- src/Parsers/ASTShowTablesQuery.h | 2 + 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 09c617e12ec..ef7fd840ac5 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -37,7 +37,11 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (!query.like.empty()) { - rewritten_query << " WHERE name " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + rewritten_query + << " WHERE name " + << (query.not_like ? "NOT " : "") + << (query.case_insensitive_like ? "ILIKE " : "LIKE ") + << std::quoted(query.like, '\''); } if (query.limit_length) @@ -54,7 +58,11 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (!query.like.empty()) { - rewritten_query << " WHERE cluster " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + rewritten_query + << " WHERE cluster " + << (query.not_like ? "NOT " : "") + << (query.case_insensitive_like ? "ILIKE " : "LIKE ") + << std::quoted(query.like, '\''); } if (query.limit_length) @@ -98,7 +106,11 @@ String InterpreterShowTablesQuery::getRewrittenQuery() rewritten_query << "database = " << std::quoted(database, '\''); if (!query.like.empty()) - rewritten_query << " AND name " << (query.not_like ? "NOT " : "") << "LIKE " << std::quoted(query.like, '\''); + rewritten_query + << " AND name " + << (query.not_like ? "NOT " : "") + << (query.case_insensitive_like ? "ILIKE " : "LIKE ") + << std::quoted(query.like, '\''); else if (query.where_expression) rewritten_query << " AND (" << query.where_expression << ")"; diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index ce44d2b56de..b59ba07d03e 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -13,43 +13,41 @@ ASTPtr ASTShowTablesQuery::clone() const return res; } +void ASTShowTablesQuery::formatLike(const FormatSettings & settings) const +{ + if (!like.empty()) + settings.ostr + << (settings.hilite ? hilite_keyword : "") + << (not_like ? " NOT" : "") + << (case_insensitive_like ? " ILIKE " : " LIKE ") + << (settings.hilite ? hilite_none : "") + << std::quoted(like, '\''); +} + +void ASTShowTablesQuery::formatLimit(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + if (limit_length) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); + limit_length->formatImpl(settings, state, frame); + } +} + void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { if (databases) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW DATABASES" << (settings.hilite ? hilite_none : ""); + formatLike(settings); + formatLimit(settings, state, frame); - if (!like.empty()) - settings.ostr - << (settings.hilite ? hilite_keyword : "") - << (not_like ? " NOT" : "") - << (case_insensitive_like ? " ILIKE " : " LIKE ") - << (settings.hilite ? hilite_none : "") - << std::quoted(like, '\''); - - if (limit_length) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); - } } else if (clusters) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW CLUSTERS" << (settings.hilite ? hilite_none : ""); + formatLike(settings); + formatLimit(settings, state, frame); - if (!like.empty()) - settings.ostr - << (settings.hilite ? hilite_keyword : "") - << (not_like ? " NOT" : "") - << (case_insensitive_like ? " ILIKE " : " LIKE ") - << (settings.hilite ? hilite_none : "") - << std::quoted(like, '\''); - - if (limit_length) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); - } } else if (cluster) { @@ -65,25 +63,15 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from); - if (!like.empty()) - settings.ostr - << (settings.hilite ? hilite_keyword : "") - << (not_like ? " NOT" : "") - << (case_insensitive_like ? " ILIKE " : " LIKE ") - << (settings.hilite ? hilite_none : "") - << std::quoted(like, '\''); + formatLike(settings); - else if (where_expression) + if (where_expression) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : ""); where_expression->formatImpl(settings, state, frame); } - if (limit_length) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " LIMIT " << (settings.hilite ? hilite_none : ""); - limit_length->formatImpl(settings, state, frame); - } + formatLimit(settings, state, frame); } } diff --git a/src/Parsers/ASTShowTablesQuery.h b/src/Parsers/ASTShowTablesQuery.h index acf365be91a..43976e8a958 100644 --- a/src/Parsers/ASTShowTablesQuery.h +++ b/src/Parsers/ASTShowTablesQuery.h @@ -36,6 +36,8 @@ public: ASTPtr clone() const override; protected: + void formatLike(const FormatSettings & settings) const; + void formatLimit(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const; void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; From c03a9487ca542b93fa51db1a671f1f365da0c081 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 9 Sep 2020 14:55:20 +0300 Subject: [PATCH 033/390] another test + fix --- src/Columns/ColumnNullable.cpp | 11 ++--- .../01457_order_by_limit.reference | 40 +++++++++++++++++++ .../0_stateless/01457_order_by_limit.sql | 30 ++++++++++++++ 3 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01457_order_by_limit.reference create mode 100644 tests/queries/0_stateless/01457_order_by_limit.sql diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 0b1c306092a..bdbc941c1e7 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -353,13 +353,8 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire size_t write_idx = first; size_t end_idx = last; - size_t current_limit = end_idx; - if (limit && limit >= read_idx && limit <= end_idx) - current_limit = limit; - - /// We simply check the limit not to do extra work. - /// Since interval begins from `first`, not from zero, we add `first` to the right side of the inequality. - while (read_idx < current_limit && !isNullAt(res[read_idx])) + /// We can't check the limit here because the interval is not sorted by nested column. + while (read_idx < end_idx && !isNullAt(res[read_idx])) { ++read_idx; ++write_idx; @@ -377,7 +372,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Relative order of NULL elements could be changed, /// but relative order of non-NULLs is preserved. - while (read_idx < end_idx && write_idx < current_limit) + while (read_idx < end_idx && write_idx < end_idx) { if (!isNullAt(res[read_idx])) { diff --git a/tests/queries/0_stateless/01457_order_by_limit.reference b/tests/queries/0_stateless/01457_order_by_limit.reference new file mode 100644 index 00000000000..348bc7ad1d4 --- /dev/null +++ b/tests/queries/0_stateless/01457_order_by_limit.reference @@ -0,0 +1,40 @@ +asc nulls last, asc +1 1 +1 2 +1 3 +1 4 +asc nulls first, asc +1 1 +1 2 +1 3 +1 4 +desc nulls last, asc +1 1 +1 2 +1 3 +1 4 +desc nulls first, asc +1 1 +1 2 +1 3 +1 4 +asc nulls last, desc +1 8 +1 7 +1 6 +1 5 +asc nulls first, desc +1 8 +1 7 +1 6 +1 5 +desc nulls last, desc +1 8 +1 7 +1 6 +1 5 +desc nulls first, desc +1 8 +1 7 +1 6 +1 5 diff --git a/tests/queries/0_stateless/01457_order_by_limit.sql b/tests/queries/0_stateless/01457_order_by_limit.sql new file mode 100644 index 00000000000..514aaeac4ab --- /dev/null +++ b/tests/queries/0_stateless/01457_order_by_limit.sql @@ -0,0 +1,30 @@ +drop table if exists order_by_another; + +create table order_by_another (a Nullable(UInt64), b UInt64) Engine = MergeTree order by tuple(); +insert into order_by_another values (1, 8), (1, 7), (1, 6), (1, 5), (1, 4), (1, 3), (1, 2), (1, 1); + +select 'asc nulls last, asc'; +select a, b from order_by_another order by a asc nulls last, b asc limit 4; + +select 'asc nulls first, asc'; +select a, b from order_by_another order by a asc nulls first, b asc limit 4; + +select 'desc nulls last, asc'; +select a, b from order_by_another order by a desc nulls last, b asc limit 4; + +select 'desc nulls first, asc'; +select a, b from order_by_another order by a desc nulls first, b asc limit 4; + +select 'asc nulls last, desc'; +select a, b from order_by_another order by a asc nulls last, b desc limit 4; + +select 'asc nulls first, desc'; +select a, b from order_by_another order by a asc nulls first, b desc limit 4; + +select 'desc nulls last, desc'; +select a, b from order_by_another order by a desc nulls last, b desc limit 4; + +select 'desc nulls first, desc'; +select a, b from order_by_another order by a desc nulls first, b desc limit 4; + +drop table if exists order_by_another; \ No newline at end of file From 063bcf6ff73d620550cbb2e2f77a7519abdcd665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=B3=E9=98=B3?= <260893248@qq.com> Date: Wed, 9 Sep 2020 20:03:26 +0800 Subject: [PATCH 034/390] =?UTF-8?q?"=E5=9B=9E=E8=AF=9D"=20->=20"=E7=9A=84?= =?UTF-8?q?=E4=BC=9A=E8=AF=9D"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix typo in Chinese --- docs/zh/introduction/history.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/introduction/history.md b/docs/zh/introduction/history.md index 1871bd75588..29c8c263f9f 100644 --- a/docs/zh/introduction/history.md +++ b/docs/zh/introduction/history.md @@ -13,7 +13,7 @@ Yandex.Metrica基于用户定义的字段，对实时访问、连接会话，生 ClickHouse还被使用在： -- 存储来自Yandex.Metrica回话重放数据。 +- 存储来自Yandex.Metrica的会话重放数据。 - 处理中间数据 - 与Analytics一起构建全球报表。 - 为调试Yandex.Metrica引擎运行查询 From 530057e79f9dad7467d9879f4d8f4420768af669 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 9 Sep 2020 15:27:05 +0300 Subject: [PATCH 035/390] update all columns --- src/Columns/ColumnDecimal.cpp | 28 +++++++++++++------ src/Columns/ColumnFixedString.cpp | 28 +++++++++++++------ src/Columns/ColumnLowCardinality.cpp | 28 +++++++++++++------ src/Columns/ColumnString.cpp | 42 +++++++++++++++++----------- src/Columns/ColumnTuple.cpp | 15 +++++----- src/Columns/ColumnUnique.h | 12 ++++---- src/Columns/ColumnVector.cpp | 6 ++-- 7 files changed, 99 insertions(+), 60 deletions(-) diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 6bab4228e9d..a899099ca4f 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -7,6 +7,7 @@ #include #include +#include #include @@ -142,28 +143,31 @@ void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn: } template -void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const +void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; - if (limit >= data.size() || limit >= equal_range.back().second) + if (limit >= data.size() || limit >= equal_ranges.back().second) limit = 0; - size_t n = equal_range.size(); + size_t number_of_ranges = equal_ranges.size(); if (limit) - --n; + --number_of_ranges; EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) + SCOPE_EXIT({equal_ranges = std::move(new_ranges);}); + + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto& [first, last] = equal_range[i]; + const auto& [first, last] = equal_ranges[i]; if (reverse) std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); + auto new_first = first; for (auto j = first + 1; j < last; ++j) { @@ -181,13 +185,20 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum if (limit) { - const auto& [first, last] = equal_range.back(); + const auto& [first, last] = equal_ranges.back(); + + if (limit < first || limit >= last) + return; + + /// Since then we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] > data[b]; }); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this](size_t a, size_t b) { return data[a] < data[b]; }); + auto new_first = first; for (auto j = first + 1; j < limit; ++j) { @@ -211,7 +222,6 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum if (new_last - new_first > 1) new_ranges.emplace_back(new_first, new_last); } - equal_range = std::move(new_ranges); } template diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index c10caa37b28..9188a988198 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -9,6 +9,8 @@ #include #include +#include + #include #include @@ -168,27 +170,29 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir } } -void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const +void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; - if (limit >= size() || limit >= equal_range.back().second) + if (limit >= size() || limit >= equal_ranges.back().second) limit = 0; - size_t k = equal_range.size(); + size_t number_of_ranges = equal_ranges.size(); if (limit) - --k; + --number_of_ranges; EqualRanges new_ranges; + SCOPE_EXIT({equal_ranges = std::move(new_ranges);}); - for (size_t i = 0; i < k; ++i) + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto& [first, last] = equal_range[i]; + const auto& [first, last] = equal_ranges[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, less(*this)); else std::sort(res.begin() + first, res.begin() + last, less(*this)); + auto new_first = first; for (auto j = first + 1; j < last; ++j) { @@ -205,11 +209,18 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu } if (limit) { - const auto& [first, last] = equal_range.back(); + const auto& [first, last] = equal_ranges.back(); + + if (limit < first || limit >= last) + return; + + /// Since then we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + auto new_first = first; for (auto j = first + 1; j < limit; ++j) { @@ -233,7 +244,6 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu if (new_last - new_first > 1) new_ranges.emplace_back(new_first, new_last); } - equal_range = std::move(new_ranges); } void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 0613e5e2b71..b6f1aede190 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -6,6 +6,7 @@ #include #include +#include namespace DB { @@ -329,22 +330,24 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di } } -void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const +void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; - if (limit >= size() || limit >= equal_range.back().second) + if (limit >= size() || limit >= equal_ranges.back().second) limit = 0; - size_t n = equal_range.size(); + size_t number_of_ranges = equal_ranges.size(); if (limit) - --n; + --number_of_ranges; EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) + SCOPE_EXIT({equal_ranges = std::move(new_ranges);}); + + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto& [first, last] = equal_range[i]; + const auto& [first, last] = equal_ranges[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); @@ -369,7 +372,13 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan if (limit) { - const auto& [first, last] = equal_range.back(); + const auto& [first, last] = equal_ranges.back(); + + if (limit < first || limit >= last) + return; + + /// Since then we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; }); @@ -377,6 +386,7 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b) {return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; }); auto new_first = first; + for (auto j = first + 1; j < limit; ++j) { if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0) @@ -387,6 +397,7 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan new_first = j; } } + auto new_last = limit; for (auto j = limit; j < last; ++j) { @@ -399,7 +410,6 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan if (new_last - new_first > 1) new_ranges.emplace_back(new_first, new_last); } - equal_range = std::move(new_ranges); } std::vector ColumnLowCardinality::scatter(ColumnIndex num_columns, const Selector & selector) const diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 57795535a64..449465e7146 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -9,7 +9,7 @@ #include #include - +#include namespace DB { @@ -325,28 +325,30 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio } } -void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges & equal_range) const +void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; - if (limit >= size() || limit > equal_range.back().second) + if (limit >= size() || limit > equal_ranges.back().second) limit = 0; EqualRanges new_ranges; - auto less_true = less(*this); - auto less_false = less(*this); - size_t n = equal_range.size(); - if (limit) - --n; + SCOPE_EXIT({equal_ranges = std::move(new_ranges);}); - for (size_t i = 0; i < n; ++i) + size_t number_of_ranges = equal_ranges.size(); + if (limit) + --number_of_ranges; + + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto &[first, last] = equal_range[i]; + const auto & [first, last] = equal_ranges[i]; + if (reverse) - std::sort(res.begin() + first, res.begin() + last, less_false); + std::sort(res.begin() + first, res.begin() + last, less(*this)); else - std::sort(res.begin() + first, res.begin() + last, less_true); + std::sort(res.begin() + first, res.begin() + last, less(*this)); + size_t new_first = first; for (size_t j = first + 1; j < last; ++j) { @@ -366,11 +368,18 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc if (limit) { - const auto &[first, last] = equal_range.back(); + const auto & [first, last] = equal_ranges.back(); + + if (limit < first || limit >= last) + return; + + /// Since then we are working inside the interval. + if (reverse) - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_false); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); else - std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_true); + std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this)); + size_t new_first = first; for (size_t j = first + 1; j < limit; ++j) { @@ -397,7 +406,6 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc if (new_last - new_first > 1) new_ranges.emplace_back(new_first, new_last); } - equal_range = std::move(new_ranges); } ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 09c7472b22b..98a6611edb7 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -344,18 +344,19 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h } } -void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const +void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; - for (const auto& column : columns) + for (const auto & column : columns) { - column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); - while (limit && !equal_range.empty() && limit <= equal_range.back().first) - equal_range.pop_back(); + column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_ranges); - if (equal_range.empty()) + while (limit && !equal_ranges.empty() && limit <= equal_ranges.back().first) + equal_ranges.pop_back(); + + if (equal_ranges.empty()) break; } } diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 59febe52112..c453af78bb1 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -387,15 +387,15 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } template -void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const +void ColumnUnique::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - if (equal_range.empty()) + if (equal_ranges.empty()) return; bool found_null_value_index = false; - for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i) + for (size_t i = 0; i < equal_ranges.size() && !found_null_value_index; ++i) { - auto& [first, last] = equal_range[i]; + auto & [first, last] = equal_ranges[i]; for (auto j = first; j < last; ++j) { if (res[j] == getNullValueIndex()) @@ -412,14 +412,14 @@ void ColumnUnique::updatePermutation(bool reverse, size_t limit, int } if (last - first <= 1) { - equal_range.erase(equal_range.begin() + i); + equal_ranges.erase(equal_ranges.begin() + i); } found_null_value_index = true; break; } } } - getNestedColumn()->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range); + getNestedColumn()->updatePermutation(reverse, limit, nan_direction_hint, res, equal_ranges); } template diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index d950c03d49b..9582aa7278a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -15,10 +15,10 @@ #include #include #include -#include -#include - #include +#include + + #if !defined(ARCADIA_BUILD) # include From f04d67b6888421d4a84c023fcbadc3779ac180fe Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 9 Sep 2020 15:41:51 +0300 Subject: [PATCH 036/390] better --- src/Columns/ColumnLowCardinality.cpp | 2 +- src/Columns/ColumnString.cpp | 26 +++++++++++++++++--------- src/Columns/ColumnUnique.h | 2 +- src/Interpreters/sortBlock.cpp | 8 ++------ 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index b6f1aede190..66a1a009ae9 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -372,7 +372,7 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan if (limit) { - const auto& [first, last] = equal_ranges.back(); + const auto & [first, last] = equal_ranges.back(); if (limit < first || limit >= last) return; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 449465e7146..b714776aa04 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -545,19 +545,22 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r } } -void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation &res, EqualRanges &equal_range) const +void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_ranges) const { - if (limit >= size() || limit >= equal_range.back().second) + if (limit >= size() || limit >= equal_ranges.back().second) limit = 0; - size_t n = equal_range.size(); + size_t number_of_ranges = equal_ranges.size(); if (limit) - --n; + --number_of_ranges; EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) + SCOPE_EXIT({equal_ranges = std::move(new_ranges);}); + + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto& [first, last] = equal_range[i]; + const auto& [first, last] = equal_ranges[i]; + if (reverse) std::sort(res.begin() + first, res.begin() + last, lessWithCollation(*this, collator)); else @@ -577,16 +580,22 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo } if (last - new_first > 1) new_ranges.emplace_back(new_first, last); - } if (limit) { - const auto& [first, last] = equal_range.back(); + const auto & [first, last] = equal_ranges.back(); + + if (limit < first || limit >= last) + return; + + /// Since then we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); else std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation(*this, collator)); + auto new_first = first; for (auto j = first + 1; j < limit; ++j) { @@ -614,7 +623,6 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo if (new_last - new_first > 1) new_ranges.emplace_back(new_first, new_last); } - equal_range = std::move(new_ranges); } void ColumnString::protect() diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index c453af78bb1..d87fdd65d15 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -382,7 +382,7 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } } - auto & column_unique = static_cast(rhs); + const auto & column_unique = static_cast(rhs); return getNestedColumn()->compareAt(n, m, *column_unique.getNestedColumn(), nan_direction_hint); } diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index cb3c36e5356..d84708b9c57 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -211,13 +211,11 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) for (const auto & column : columns_with_sort_desc) { while (!ranges.empty() && limit && limit <= ranges.back().first) - { ranges.pop_back(); - } + if (ranges.empty()) - { break; - } + column.column->updatePermutation( column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); } @@ -225,9 +223,7 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) size_t columns = block.columns(); for (size_t i = 0; i < columns; ++i) - { block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit); - } } } From ead6bfe05cbd6ce134ddd0ed370702f96ed864e7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 9 Sep 2020 15:42:56 +0300 Subject: [PATCH 037/390] better[2] --- src/Columns/ColumnString.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index b714776aa04..a3da1d745c1 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -547,6 +547,9 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_ranges) const { + if (equal_ranges.empty()) + return; + if (limit >= size() || limit >= equal_ranges.back().second) limit = 0; From 751e4109152671ece957a0c36af43f0e909f41d7 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 9 Sep 2020 17:11:08 +0300 Subject: [PATCH 038/390] better[3] --- src/Columns/ColumnArray.cpp | 16 +++++++++++----- src/Columns/ColumnDecimal.cpp | 4 ++-- src/Columns/ColumnFixedString.cpp | 4 ++-- src/Columns/ColumnLowCardinality.cpp | 2 +- src/Columns/ColumnString.cpp | 4 ++-- src/Columns/ColumnVector.cpp | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index cd4aa57c18f..e4d17c586ac 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -787,15 +787,15 @@ void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_directio if (limit >= size() || limit >= equal_range.back().second) limit = 0; - size_t n = equal_range.size(); + size_t number_of_ranges = equal_range.size(); if (limit) - --n; + --number_of_ranges; EqualRanges new_ranges; - for (size_t i = 0; i < n; ++i) + for (size_t i = 0; i < number_of_ranges; ++i) { - const auto& [first, last] = equal_range[i]; + const auto & [first, last] = equal_range[i]; if (reverse) std::sort(res.begin() + first, res.begin() + last, Less(*this, nan_direction_hint)); @@ -820,7 +820,13 @@ void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_directio if (limit) { - const auto& [first, last] = equal_range.back(); + const auto & [first, last] = equal_range.back(); + + if (limit < first || limit > last) + return; + + /// Since then we are working inside the interval. + if (reverse) std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less(*this, nan_direction_hint)); else diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index a899099ca4f..13c7ea46fbc 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -185,9 +185,9 @@ void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColum if (limit) { - const auto& [first, last] = equal_ranges.back(); + const auto & [first, last] = equal_ranges.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then we are working inside the interval. diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 9188a988198..0e44b83791c 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -209,9 +209,9 @@ void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permu } if (limit) { - const auto& [first, last] = equal_ranges.back(); + const auto & [first, last] = equal_ranges.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then we are working inside the interval. diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 66a1a009ae9..dd1e2f60399 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -374,7 +374,7 @@ void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan { const auto & [first, last] = equal_ranges.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then we are working inside the interval. diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index a3da1d745c1..23414626a59 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -370,7 +370,7 @@ void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direc { const auto & [first, last] = equal_ranges.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then we are working inside the interval. @@ -589,7 +589,7 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, boo { const auto & [first, last] = equal_ranges.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then we are working inside the interval. diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 9582aa7278a..c548ce3ca5c 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -282,7 +282,7 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire { const auto & [first, last] = equal_range.back(); - if (limit < first || limit >= last) + if (limit < first || limit > last) return; /// Since then, we are working inside the interval. From b8d28c864c8902c2e2aa270112f3308d90a2a490 Mon Sep 17 00:00:00 2001 From: Gao Qiang <30835199+dreamerfable@users.noreply.github.com> Date: Wed, 9 Sep 2020 22:45:49 +0800 Subject: [PATCH 039/390] Update aggregatingmergetree.md --- .../mergetree-family/aggregatingmergetree.md | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/zh/engines/table-engines/mergetree-family/aggregatingmergetree.md index e931b6f6710..03825a41f95 100644 --- a/docs/zh/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,12 +1,15 @@ # AggregatingMergeTree {#aggregatingmergetree} -该引擎继承自 [MergeTree](mergetree.md)，并改变了数据片段的合并逻辑。 ClickHouse 会将相同主键的所有行（在一个数据片段内）替换为单个存储一系列聚合函数状态的行。 +该引擎继承自 [MergeTree](mergetree.md)，并改变了数据片段的合并逻辑。 ClickHouse 会将一个数据片段内所有具有相同主键（准确的说是 [排序键](../../../engines/table-engines/mergetree-family/mergetree.md)）的行替换成一行，这一行会存储一系列聚合函数的状态。 -可以使用 `AggregatingMergeTree` 表来做增量数据统计聚合，包括物化视图的数据聚合。 +可以使用 `AggregatingMergeTree` 表来做增量数据的聚合统计，包括物化视图的数据聚合。 -引擎需使用 [AggregateFunction](../../../engines/table-engines/mergetree-family/aggregatingmergetree.md) 类型来处理所有列。 +引擎使用以下类型来处理所有列： -如果要按一组规则来合并减少行数，则使用 `AggregatingMergeTree` 是合适的。 +- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) + +`AggregatingMergeTree` 适用于能够按照一定的规则缩减行数的情况。 ## 建表 {#jian-biao} @@ -20,10 +23,11 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] +[TTL expr] [SETTINGS name=value, ...] ``` -语句参数的说明，请参阅 [语句描述](../../../engines/table-engines/mergetree-family/aggregatingmergetree.md)。 +语句参数的说明，请参阅 [建表语句描述](../../../sql-reference/statements/create.md#create-table-query)。 **子句** @@ -33,7 +37,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 已弃用的建表方法 -!!! 注意 "注意" +!!! attention "注意" 不要在新项目中使用该方法，可能的话，请将旧项目切换到上述方法。 ``` sql @@ -45,15 +49,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE [=] AggregatingMergeTree(date-column [, sampling_expression], (primary, key), index_granularity) ``` -上面的所有参数跟 `MergeTree` 中的一样。 +上面的所有参数的含义跟 `MergeTree` 中的一样。 ## SELECT 和 INSERT {#select-he-insert} -插入数据，需使用带有聚合 -State- 函数的 [INSERT SELECT](../../../engines/table-engines/mergetree-family/aggregatingmergetree.md) 语句。 +要插入数据，需使用带有 -State- 聚合函数的 [INSERT SELECT](../../../sql-reference/statements/insert-into.md) 语句。 从 `AggregatingMergeTree` 表中查询数据时，需使用 `GROUP BY` 子句并且要使用与插入时相同的聚合函数，但后缀要改为 `-Merge` 。 -在 `SELECT` 查询的结果中，对于 ClickHouse 的所有输出格式 `AggregateFunction` 类型的值都实现了特定的二进制表示法。如果直接用 `SELECT` 导出这些数据，例如如用 `TabSeparated` 格式，那么这些导出数据也能直接用 `INSERT` 语句加载导入。 +对于 `SELECT` 查询的结果， `AggregateFunction` 类型的值对 ClickHouse 的所有输出格式都实现了特定的二进制表示法。在进行数据转储时，例如使用 `TabSeparated` 格式进行 `SELECT` 查询，那么这些转储数据也能直接用 `INSERT` 语句导回。 ## 聚合物化视图的示例 {#ju-he-wu-hua-shi-tu-de-shi-li} From 6ed3f7fb86b19b94fa236e5c2659ea7a217357b3 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Wed, 9 Sep 2020 23:42:03 +0800 Subject: [PATCH 040/390] ISSUES-4006 try fix integration test failure when mysql 5.7 --- tests/integration/test_materialize_mysql_database/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index c6b3810fc74..7affa665cd3 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -89,7 +89,7 @@ def started_mysql_8_0(): def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7): materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1") - materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql5_7") + materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql1") def test_materialize_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0): From e91d120e1a9c8326c895fcb45c9e43f5cdedfdd1 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 9 Sep 2020 20:11:23 +0300 Subject: [PATCH 041/390] Bump CI. From b9bf67b6ac31741246dce7790ec3890fd599ff7d Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 10 Sep 2020 04:27:36 +0300 Subject: [PATCH 042/390] allow to randomize part type in MergeTree --- src/Storages/MergeTree/MergeTreeSettings.h | 5 ++- .../MergeTree/registerStorageMergeTree.cpp | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 49847617d30..9adbc1a7b3d 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -20,7 +20,7 @@ struct Settings; M(UInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ \ /** Data storing format settings. */ \ - M(UInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ + M(UInt64, min_bytes_for_wide_part, 10485760, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ M(UInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \ M(UInt64, min_bytes_for_compact_part, 0, "Experimental. Minimal uncompressed size in bytes to create part in compact format instead of saving it in RAM", 0) \ M(UInt64, min_rows_for_compact_part, 0, "Experimental. Minimal number of rows to create part in compact format instead of saving it in RAM", 0) \ @@ -97,6 +97,9 @@ struct Settings; M(String, storage_policy, "default", "Name of storage disk policy", 0) \ M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ \ + /** Settings for testing purposes */ \ + M(Bool, randomize_part_type, false, "For testing purposes only. Randomizes part type between wide and compact", 0) \ + \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ M(UInt64, check_delay_period, 60, "Obsolete setting, does nothing.", 0) \ diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index b0c422bd79f..5609c130aba 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -233,6 +234,25 @@ If you use the Replicated version of engines, see https://clickhouse.tech/docs/e } +static void randomizePartTypeSettings(const std::unique_ptr & storage_settings) +{ + static constexpr auto MAX_THRESHOLD_FOR_ROWS = 100000; + static constexpr auto MAX_THRESHOLD_FOR_BYTES = 1024 * 1024 * 10; + + /// Create all parts in wide format with probability 1/3. + if (thread_local_rng() % 3 == 0) + { + storage_settings->min_rows_for_wide_part = 0; + storage_settings->min_bytes_for_wide_part = 0; + } + else + { + storage_settings->min_rows_for_wide_part = std::uniform_int_distribution{0, MAX_THRESHOLD_FOR_ROWS}(thread_local_rng); + storage_settings->min_bytes_for_wide_part = std::uniform_int_distribution{0, MAX_THRESHOLD_FOR_BYTES}(thread_local_rng); + } +} + + static StoragePtr create(const StorageFactory::Arguments & args) { /** [Replicated][|Summing|Collapsing|Aggregating|Replacing|Graphite]MergeTree (2 * 7 combinations) engines @@ -652,6 +672,20 @@ static StoragePtr create(const StorageFactory::Arguments & args) ++arg_num; } + /// Allow to randomize part type for tests to cover more cases. + /// But if settings were set explicitly restrict it. + if (storage_settings->randomize_part_type + && !storage_settings->min_rows_for_wide_part.changed + && !storage_settings->min_bytes_for_wide_part.changed) + { + randomizePartTypeSettings(storage_settings); + LOG_INFO(&Poco::Logger::get(args.table_id.getNameForLogs() + " (registerStorageMergeTree)"), + "Applied setting 'randomize_part_type'. " + "Setting 'min_rows_for_wide_part' changed to {}. " + "Setting 'min_bytes_for_wide_part' changed to {}.", + storage_settings->min_rows_for_wide_part, storage_settings->min_bytes_for_wide_part); + } + if (arg_num != arg_cnt) throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS); From a173a863a8e971732524cceeb85de1dbf03d1b11 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Thu, 10 Sep 2020 11:03:35 +0800 Subject: [PATCH 043/390] fix style error --- src/Parsers/ASTShowTablesQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parsers/ASTShowTablesQuery.cpp b/src/Parsers/ASTShowTablesQuery.cpp index b59ba07d03e..1e8dad13ad3 100644 --- a/src/Parsers/ASTShowTablesQuery.cpp +++ b/src/Parsers/ASTShowTablesQuery.cpp @@ -63,7 +63,7 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(from); - formatLike(settings); + formatLike(settings); if (where_expression) { @@ -71,7 +71,7 @@ void ASTShowTablesQuery::formatQueryImpl(const FormatSettings & settings, Format where_expression->formatImpl(settings, state, frame); } - formatLimit(settings, state, frame); + formatLimit(settings, state, frame); } } From fea763bb751fd0fb4abfef9ff34acdabb8b8e0d8 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 10 Sep 2020 11:37:00 +0800 Subject: [PATCH 044/390] ISSUES-4006 support decimal 256 for mysql block input stream --- src/Core/ExternalResultDescription.cpp | 2 ++ src/Core/ExternalResultDescription.h | 3 ++- src/Formats/MySQLBlockInputStream.cpp | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Core/ExternalResultDescription.cpp b/src/Core/ExternalResultDescription.cpp index 941ee003c94..7165d73b7d0 100644 --- a/src/Core/ExternalResultDescription.cpp +++ b/src/Core/ExternalResultDescription.cpp @@ -74,6 +74,8 @@ void ExternalResultDescription::init(const Block & sample_block_) types.emplace_back(ValueType::vtDecimal64, is_nullable); else if (typeid_cast *>(type)) types.emplace_back(ValueType::vtDecimal128, is_nullable); + else if (typeid_cast *>(type)) + types.emplace_back(ValueType::vtDecimal256, is_nullable); else throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE}; } diff --git a/src/Core/ExternalResultDescription.h b/src/Core/ExternalResultDescription.h index 29294fcf2c8..f8ba2a6bba2 100644 --- a/src/Core/ExternalResultDescription.h +++ b/src/Core/ExternalResultDescription.h @@ -29,7 +29,8 @@ struct ExternalResultDescription vtDateTime64, vtDecimal32, vtDecimal64, - vtDecimal128 + vtDecimal128, + vtDecimal256 }; Block sample_block; diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index f85680c0031..be1e254b22f 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -90,7 +90,8 @@ namespace case ValueType::vtDateTime64:[[fallthrough]]; case ValueType::vtDecimal32: [[fallthrough]]; case ValueType::vtDecimal64: [[fallthrough]]; - case ValueType::vtDecimal128: + case ValueType::vtDecimal128:[[fallthrough]]; + case ValueType::vtDecimal256: { ReadBuffer buffer(const_cast(value.data()), value.size(), 0); data_type.deserializeAsWholeText(column, buffer, FormatSettings{}); From 338b69201d126a9ccc0be828b6fed4f6a0affef9 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 10 Sep 2020 17:20:28 +0800 Subject: [PATCH 045/390] ISSUES-4006 add test decimal when dump stage --- .../materialize_with_ddl.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index fa31c43e6c1..a953202bff0 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -98,19 +98,29 @@ def dml_with_materialize_mysql_database(clickhouse_node, mysql_node, service_nam def materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) - mysql_node.query("CREATE TABLE test_database.test_table_1 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), " - "_decimal DECIMAL(65, 30)) ENGINE = InnoDB;") + mysql_node.query("CREATE TABLE test_database.test_table_1 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;") mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + ('9' * 35) + "." + ('9' * 30) + ")") mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + ('0' * 29) + "1)") mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + ('9' * 35) + "." + ('9' * 30) + ")") mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") + + clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + ('0' * 29) + "1\n") + + mysql_node.query("CREATE TABLE test_database.test_table_2 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;") + mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + ('9' * 35) + "." + ('9' * 30) + ")") + mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + ('0' * 29) + "1)") + mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + ('9' * 35) + "." + ('9' * 30) + ")") + mysql_node.query("INSERT INTO test_database.test_table_2 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") + check_query(clickhouse_node, "SELECT * FROM test_database.test_table_2 ORDER BY key FORMAT TSV", + "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" + "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" + "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" + "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + ('0' * 29) + "1\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") From 223fc4d1e7b0ffd6ad7f2b2226d2210eec5af64d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 10 Sep 2020 12:59:10 +0300 Subject: [PATCH 046/390] Avoid error while building the report on broken perf tests --- docker/test/performance-comparison/compare.sh | 12 ++++++++++++ src/Interpreters/ExpressionActions.cpp | 12 ++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 364e9994ab7..2ae7910dcaa 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -394,12 +394,24 @@ create table query_run_metrics_denorm engine File(TSV, 'analyze/query-run-metric order by test, query_index, metric_names, version, query_id ; +-- Filter out tests that don't have an even number of runs, to avoid breaking +-- the further calculations. This may happen if there was an error during the +-- test runs, e.g. the server died. It will be reported in test errors, so we +-- don't have to report it again. +create view broken_tests as + select test_name + from query_runs + group by test_name + having count(*) % 2 == 0 + ; + -- This is for statistical processing with eqmed.sql create table query_run_metrics_for_stats engine File( TSV, -- do not add header -- will parse with grep 'analyze/query-run-metrics-for-stats.tsv') as select test, query_index, 0 run, version, metric_values from query_run_metric_arrays + where test not in broken_tests order by test, query_index, run, version ; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 33fa6215160..0c287e4026d 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -607,8 +607,16 @@ void ExpressionActions::execute(Block & block, bool dry_run) const { for (const auto & action : actions) { - action.execute(block, dry_run); - checkLimits(block); + try + { + action.execute(block, dry_run); + checkLimits(block); + } + catch (Exception & e) + { + e.addMessage(fmt::format("while executing '{}'", action.toString())); + throw; + } } } From a2a647eb1caac92b13f73b04651c7d64b66c0fc1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 10 Sep 2020 13:02:45 +0300 Subject: [PATCH 047/390] fixup --- docker/test/performance-comparison/compare.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 2ae7910dcaa..08f4cb599ab 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -398,11 +398,11 @@ create table query_run_metrics_denorm engine File(TSV, 'analyze/query-run-metric -- the further calculations. This may happen if there was an error during the -- test runs, e.g. the server died. It will be reported in test errors, so we -- don't have to report it again. -create view broken_tests as - select test_name +create view broken_queries as + select test, query_index from query_runs - group by test_name - having count(*) % 2 == 0 + group by test, query_index + having count(*) % 2 != 0 ; -- This is for statistical processing with eqmed.sql @@ -411,7 +411,7 @@ create table query_run_metrics_for_stats engine File( 'analyze/query-run-metrics-for-stats.tsv') as select test, query_index, 0 run, version, metric_values from query_run_metric_arrays - where test not in broken_tests + where (test, query_index) not in broken_queries order by test, query_index, run, version ; From 513e0043f73c51e67454a5dedbe8d9f65d85da72 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 10 Sep 2020 18:57:18 +0300 Subject: [PATCH 048/390] initial docs header --- cmake_files_header.md | 137 ++++++++++++++++++++++++++++++++++++ cmake_flags_md_generator.sh | 1 + 2 files changed, 138 insertions(+) create mode 100644 cmake_files_header.md create mode 100644 cmake_flags_md_generator.sh diff --git a/cmake_files_header.md b/cmake_files_header.md new file mode 100644 index 00000000000..bb613edaf1b --- /dev/null +++ b/cmake_files_header.md @@ -0,0 +1,137 @@ +# CMake flags + +## Developer's guide for adding new CMake options + +### Don't be obvious. Be informative. + +Bad: +``` +option (ENABLE_TESTS "Enables testing" OFF) +``` + +This description is quite useless as is neither gives the viewer any additional information nor explains the option +purpose. If the option's name is quite self-descriptive, prefer the empty description. + +Better: + +``` +option(ENABLE_TESTS OFF) +``` + +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above +the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). +The comment is parsed into a separate column (see below). + +Even better: + +``` +# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). +# see tests/CMakeLists.txt for implementation detail. +option(ENABLE_GTEST_TESTS OFF) +``` + +### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. + +Suppose you have an option that may strip debug symbols from the ClickHouse's part. +This can speed up the linking process, but produces a binary that cannot be debugged. +In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. +Also, such options should be disabled if applies. + +Bad: +``` +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions. + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() + +``` +Better: + +``` +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions." + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + message(WARNING "Not generating debugger info for ClickHouse functions") + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() +``` + +### In the option's description, explain WHAT the option does rather than WHY it does something. + +The WHY explanation should be placed in the comment. +You may find that the option's name is self-descriptive. + +Bad: + +``` +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better: + +``` +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +### Don't assume other developers know as much as you do. + +In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to +the tool's docs. It won't take much of your time. + +Bad: + +``` +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better (combined with the above hint): + +``` +# https://clang.llvm.org/docs/ThinLTO.html +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +Other example, bad: + +``` +option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) +``` + +Better: + +``` +# https://github.com/include-what-you-use/include-what-you-use +option (USE_INCLUDE_WHAT_YOU_USE OFF) +``` + +### Prefer consistent default values. + +CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. +Prefer the `ON/OFF` values, if possible. + + +## List of CMake flags + +* This list is auto-generated by [this bash script](bash.sh). +* The flag name is a link to its position in the code. + +| Name | Description | Default value | Comment | +|------|-------------|---------------|---------| +| \#0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | +| \#1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | +| \#2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | +| \#N | … | … | … | … | … | + + diff --git a/cmake_flags_md_generator.sh b/cmake_flags_md_generator.sh new file mode 100644 index 00000000000..e72f7660e6d --- /dev/null +++ b/cmake_flags_md_generator.sh @@ -0,0 +1 @@ +#!/bin/bash/ From 6031e6bae95f8207ab554a422244ecf160aa90d8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 10 Sep 2020 20:09:19 +0300 Subject: [PATCH 049/390] fix tests --- .../configs/wide_parts_only.xml | 6 +++++ .../test_adaptive_granularity/test.py | 24 ++++++++++++++----- .../configs/compact_parts.xml | 1 + .../test_polymorphic_parts/test.py | 1 + ...46_clear_column_in_partition_zookeeper.sql | 6 ++--- ...system_columns_and_system_tables.reference | 2 +- ...00753_system_columns_and_system_tables.sql | 3 ++- ...ndex_granularity_collapsing_merge_tree.sql | 2 +- ..._adaptive_index_granularity_merge_tree.sql | 12 ++-------- .../queries/0_stateless/00933_ttl_simple.sql | 1 - ...ms_in_system_parts_columns_table.reference | 2 +- ...hecksums_in_system_parts_columns_table.sql | 10 ++------ 12 files changed, 38 insertions(+), 32 deletions(-) create mode 100644 tests/integration/test_adaptive_granularity/configs/wide_parts_only.xml diff --git a/tests/integration/test_adaptive_granularity/configs/wide_parts_only.xml b/tests/integration/test_adaptive_granularity/configs/wide_parts_only.xml new file mode 100644 index 00000000000..42e2173f718 --- /dev/null +++ b/tests/integration/test_adaptive_granularity/configs/wide_parts_only.xml @@ -0,0 +1,6 @@ + + + 0 + 0 + + diff --git a/tests/integration/test_adaptive_granularity/test.py b/tests/integration/test_adaptive_granularity/test.py index 21d65588de4..7efafb4ddd1 100644 --- a/tests/integration/test_adaptive_granularity/test.py +++ b/tests/integration/test_adaptive_granularity/test.py @@ -13,10 +13,10 @@ node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml' node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', with_installed_binary=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', with_installed_binary=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) @@ -270,8 +270,14 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node): node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)") def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "1") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "1") + new_config = """ + + 1 + 0 +""" + + n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", new_config) + n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", new_config) node.restart_with_latest_version(callback_onstop=callback) node.query("SYSTEM RELOAD CONFIG") @@ -304,8 +310,14 @@ def test_version_update_two_nodes(start_dynamic_cluster): node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20) assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n' def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "0") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "0") + new_config = """ + + 0 + 0 +""" + + n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", new_config) + n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", new_config) node12.restart_with_latest_version(callback_onstop=callback) diff --git a/tests/integration/test_polymorphic_parts/configs/compact_parts.xml b/tests/integration/test_polymorphic_parts/configs/compact_parts.xml index e14c3f0ceae..5b3afe65d92 100644 --- a/tests/integration/test_polymorphic_parts/configs/compact_parts.xml +++ b/tests/integration/test_polymorphic_parts/configs/compact_parts.xml @@ -1,5 +1,6 @@ 512 + 0 diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index ed89f768d4c..cf2268bc831 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -42,6 +42,7 @@ def create_tables(name, nodes, node_settings, shard): ORDER BY id SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, min_rows_for_wide_part = {min_rows_for_wide_part}, min_rows_for_compact_part = {min_rows_for_compact_part}, + min_bytes_for_wide_part = 0, min_bytes_for_compact_part = 0, in_memory_parts_enable_wal = 1 '''.format(name=name, shard=shard, repl=i, **settings)) diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index 56f3a654682..e6de5a91ce3 100644 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -3,7 +3,7 @@ SELECT '===Ordinary case==='; SET replication_alter_partitions_sync = 2; DROP TABLE IF EXISTS clear_column; -CREATE TABLE clear_column (d Date, num Int64, str String) ENGINE = MergeTree(d, d, 8192); +CREATE TABLE clear_column (d Date, num Int64, str String) ENGINE = MergeTree ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; INSERT INTO clear_column VALUES ('2016-12-12', 1, 'a'), ('2016-11-12', 2, 'b'); @@ -24,8 +24,8 @@ SELECT '===Replicated case==='; DROP TABLE IF EXISTS clear_column1; DROP TABLE IF EXISTS clear_column2; SELECT sleep(1) FORMAT Null; -CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '1', d, d, 8192); -CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '2', d, d, 8192); +CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '1') ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '2') ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; INSERT INTO clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); SYSTEM SYNC REPLICA clear_column2; diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference b/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference index 47c15da4b22..4d1fab83cc1 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables.reference @@ -6,7 +6,7 @@ │ name2 │ 1 │ 0 │ 0 │ 0 │ │ name3 │ 0 │ 0 │ 0 │ 0 │ └───────┴─────────────────────┴───────────────────┴───────────────────┴────────────────────┘ -147 1 +231 1 ┌─name────────────────┬─partition_key─┬─sorting_key───┬─primary_key─┬─sampling_key─┐ │ check_system_tables │ date │ date, version │ date │ │ └─────────────────────┴───────────────┴───────────────┴─────────────┴──────────────┘ diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables.sql index 1d7faa32952..9b9fa04e6b0 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables.sql @@ -9,7 +9,8 @@ CREATE TABLE check_system_tables ) ENGINE = MergeTree() ORDER BY name1 PARTITION BY name2 - SAMPLE BY name1; + SAMPLE BY name1 + SETTINGS min_bytes_for_wide_part = 0; SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows FROM system.tables diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql index 74159738bce..d4c19cbe8f2 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql @@ -41,7 +41,7 @@ CREATE TABLE four_rows_per_granule ( Sign Int8 ) ENGINE CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes=110, min_index_granularity_bytes=100, write_final_mark = 0, - + min_bytes_for_wide_part = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql index f2e9e9749e3..249c6eebfcf 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql @@ -61,11 +61,7 @@ CREATE TABLE four_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -<<<<<<< HEAD -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0, min_bytes_for_wide_part = 0; -======= -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 10, write_final_mark = 0; ->>>>>>> upstream/master +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -130,11 +126,7 @@ CREATE TABLE adaptive_granularity_alter ( k UInt64, v1 UInt64, v2 Int64 -<<<<<<< HEAD -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, write_final_mark = 0, min_bytes_for_wide_part = 0; -======= -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 100, write_final_mark = 0; ->>>>>>> upstream/master +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 100, write_final_mark = 0, min_bytes_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); diff --git a/tests/queries/0_stateless/00933_ttl_simple.sql b/tests/queries/0_stateless/00933_ttl_simple.sql index 83d9962043d..c0adcd21e62 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.sql +++ b/tests/queries/0_stateless/00933_ttl_simple.sql @@ -23,7 +23,6 @@ create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 DAY) engine = Mer insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 3); -optimize table ttl_00933_1 final; select * from ttl_00933_1 order by d; drop table if exists ttl_00933_1; diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference index 3bcfc00eded..099fe566817 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.reference @@ -1 +1 @@ -20000101_20000101_1_1_0 test_00961 b78f351b7498ecc9d4732ad29c3952de 1d4b7fbf05d0fc5c2f4559ca75aa32f7 38f047b57fd1bb81cf77e273deb34218 +20000101_1_1_0 test_00961 5f2e2d4bbc14336f44037e3ac667f247 ed226557cd4e18ecf3ae06c6d5e6725c da96ff1e527a8a1f908ddf2b1d0af239 diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index 46daa0bf711..792bf62f9b1 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,21 +1,15 @@ DROP TABLE IF EXISTS test_00961; -CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree(d, (a, b), 111); +CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) + ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); SELECT -<<<<<<< HEAD name, table, hash_of_all_files, hash_of_uncompressed_files, -======= - name, - table, - hash_of_all_files, - hash_of_uncompressed_files, ->>>>>>> upstream/master uncompressed_hash_of_compressed_files FROM system.parts WHERE table = 'test_00961' and database = currentDatabase(); From a420976041581f4e0af3e9363a16a3c2aa119d97 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 10 Sep 2020 20:12:06 +0300 Subject: [PATCH 050/390] Bump CI [2]. From fa6d88b3b29f9a0e852e009651c32652e0201fad Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 10 Sep 2020 21:43:02 +0300 Subject: [PATCH 051/390] fix more tests --- .../test_filesystem_layout/test.py | 2 +- .../configs/config.d/storage_conf.xml | 4 ++++ tests/integration/test_partition/test.py | 2 +- .../configs/config.d/storage_conf.xml | 4 ++++ .../test_replicated_merge_tree_s3/test.py | 22 ++++++++++++++----- tests/integration/test_ttl_replicated/test.py | 10 +++++---- ...46_clear_column_in_partition_zookeeper.sql | 6 ++--- .../0_stateless/00933_ttl_simple.reference | 8 +++---- .../queries/0_stateless/00933_ttl_simple.sql | 15 ++++++++----- 9 files changed, 48 insertions(+), 25 deletions(-) diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 83389b3d9bd..777e5ab7b9a 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -19,7 +19,7 @@ def test_file_path_escaping(started_cluster): node.query('CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary') node.query(''' CREATE TABLE test.`T.a_b,l-e!` (`~Id` UInt32) - ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id`; + ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id` SETTINGS min_bytes_for_wide_part = 0; ''') node.query('''INSERT INTO test.`T.a_b,l-e!` VALUES (1);''') node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''') diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index d097675ca63..343f248c5fb 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -25,4 +25,8 @@ + + + 0 + diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 80fbe947316..2a2bbe205b5 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -178,7 +178,7 @@ def test_attach_check_all_parts(attach_check_all_parts_table): exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0')) error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") - assert 0 <= error.find('No columns in part 0_5_5_0') + assert 0 <= error.find('No columns in part 0_5_5_0') or 0 <= error.find('No columns.txt in part 0_5_5_0') parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0') diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml index b32770095fc..f3b7f959ce9 100644 --- a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -18,4 +18,8 @@ + + + 0 + diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index a77a69b842b..612b50becf7 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -30,7 +30,8 @@ def cluster(): FILES_OVERHEAD = 1 FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files -FILES_OVERHEAD_PER_PART = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def random_string(length): @@ -44,7 +45,7 @@ def generate_values(date_str, count, sign=1): return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) -def create_table(cluster): +def create_table(cluster, additional_settings=None): create_table_statement = """ CREATE TABLE s3_test ( dt Date, @@ -56,6 +57,9 @@ def create_table(cluster): ORDER BY (dt, id) SETTINGS storage_policy='s3' """ + if additional_settings: + create_table_statement += "," + create_table_statement += additional_settings for node in cluster.instances.values(): node.query(create_table_statement) @@ -72,9 +76,15 @@ def drop_table(cluster): for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): minio.remove_object(cluster.minio_bucket, obj.object_name) - -def test_insert_select_replicated(cluster): - create_table(cluster) +@pytest.mark.parametrize( + "min_rows_for_wide_part,files_per_part", + [ + (0, FILES_OVERHEAD_PER_PART_WIDE), + (8192, FILES_OVERHEAD_PER_PART_COMPACT) + ] +) +def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): + create_table(cluster, additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part)) all_values = "" for node_idx in range(1, 4): @@ -90,4 +100,4 @@ def test_insert_select_replicated(cluster): assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", settings={"select_sequential_consistency": 1}) == all_values minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * (FILES_OVERHEAD + FILES_OVERHEAD_PER_PART * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * (FILES_OVERHEAD + files_per_part * 3) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 0f201f569b3..39d595662d0 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -30,7 +30,7 @@ def drop_table(nodes, table_name): node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) time.sleep(1) - +# Column TTL works only with wide parts, because it's very expensive to apply it for compact parts def test_ttl_columns(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: @@ -38,7 +38,7 @@ def test_ttl_columns(started_cluster): ''' CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; + ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, min_bytes_for_wide_part=0; '''.format(replica=node.name)) node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)") @@ -59,7 +59,8 @@ def test_merge_with_ttl_timeout(started_cluster): ''' CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date); + ORDER BY id PARTITION BY toDayOfMonth(date) + SETTINGS min_bytes_for_wide_part=0; '''.format(replica=node.name, table=table)) node1.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) @@ -198,7 +199,7 @@ def test_ttl_double_delete_rule_returns_error(started_cluster): CREATE TABLE test_ttl(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) - TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0; + TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0 '''.format(replica=node1.name)) assert False except client.QueryRuntimeException: @@ -246,6 +247,7 @@ limitations under the License.""" ) ENGINE = {engine} ORDER BY tuple() TTL d1 + INTERVAL 1 DAY DELETE + SETTINGS min_bytes_for_wide_part=0 """.format(name=name, engine=engine)) node1.query("""ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format(name=name)) diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql index e6de5a91ce3..bd6c12ffce4 100644 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_zookeeper.sql @@ -3,7 +3,7 @@ SELECT '===Ordinary case==='; SET replication_alter_partitions_sync = 2; DROP TABLE IF EXISTS clear_column; -CREATE TABLE clear_column (d Date, num Int64, str String) ENGINE = MergeTree ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE clear_column (d Date, num Int64, str String) ENGINE = MergeTree ORDER BY d PARTITION by toYYYYMM(d) SETTINGS min_bytes_for_wide_part = 0; INSERT INTO clear_column VALUES ('2016-12-12', 1, 'a'), ('2016-11-12', 2, 'b'); @@ -24,8 +24,8 @@ SELECT '===Replicated case==='; DROP TABLE IF EXISTS clear_column1; DROP TABLE IF EXISTS clear_column2; SELECT sleep(1) FORMAT Null; -CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '1') ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; -CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '2') ORDER BY d PARTITION by d SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '1') ORDER BY d PARTITION by toYYYYMM(d) SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column', '2') ORDER BY d PARTITION by toYYYYMM(d) SETTINGS min_bytes_for_wide_part = 0; INSERT INTO clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); SYSTEM SYNC REPLICA clear_column2; diff --git a/tests/queries/0_stateless/00933_ttl_simple.reference b/tests/queries/0_stateless/00933_ttl_simple.reference index a4ef8033328..e3982814eab 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.reference +++ b/tests/queries/0_stateless/00933_ttl_simple.reference @@ -6,11 +6,11 @@ 2000-10-10 00:00:00 0 2100-10-10 00:00:00 3 2100-10-10 2 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() - 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() - 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() + 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL now() + 1000\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 1 1 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() - 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() - 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 1 0 -CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() + 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.ttl_00933_1\n(\n `b` Int32,\n `a` Int32 TTL today() + 1\n)\nENGINE = MergeTree\nPARTITION BY tuple()\nORDER BY tuple()\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192 1 1 diff --git a/tests/queries/0_stateless/00933_ttl_simple.sql b/tests/queries/0_stateless/00933_ttl_simple.sql index c0adcd21e62..aa8b33b2999 100644 --- a/tests/queries/0_stateless/00933_ttl_simple.sql +++ b/tests/queries/0_stateless/00933_ttl_simple.sql @@ -1,6 +1,8 @@ drop table if exists ttl_00933_1; -create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 second, b Int ttl d + interval 1 second) engine = MergeTree order by tuple() partition by toMinute(d); +-- Column TTL works only with wide parts, because it's very expensive to apply it for compact parts + +create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 second, b Int ttl d + interval 1 second) engine = MergeTree order by tuple() partition by toMinute(d) settings min_bytes_for_wide_part = 0; insert into ttl_00933_1 values (now(), 1, 2); insert into ttl_00933_1 values (now(), 3, 4); select sleep(1.1) format Null; @@ -19,10 +21,11 @@ select a, b from ttl_00933_1; drop table if exists ttl_00933_1; -create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 DAY) engine = MergeTree order by tuple() partition by toDayOfMonth(d); +create table ttl_00933_1 (d DateTime, a Int ttl d + interval 1 DAY) engine = MergeTree order by tuple() partition by toDayOfMonth(d) settings min_bytes_for_wide_part = 0; insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_1 values (toDateTime('2000-10-10 00:00:00'), 3); +optimize table ttl_00933_1 final; select * from ttl_00933_1 order by d; drop table if exists ttl_00933_1; @@ -44,7 +47,7 @@ select * from ttl_00933_1 order by d; -- const DateTime TTL positive drop table if exists ttl_00933_1; -create table ttl_00933_1 (b Int, a Int ttl now()-1000) engine = MergeTree order by tuple() partition by tuple(); +create table ttl_00933_1 (b Int, a Int ttl now()-1000) engine = MergeTree order by tuple() partition by tuple() settings min_bytes_for_wide_part = 0; show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); optimize table ttl_00933_1 final; @@ -52,7 +55,7 @@ select * from ttl_00933_1; -- const DateTime TTL negative drop table if exists ttl_00933_1; -create table ttl_00933_1 (b Int, a Int ttl now()+1000) engine = MergeTree order by tuple() partition by tuple(); +create table ttl_00933_1 (b Int, a Int ttl now()+1000) engine = MergeTree order by tuple() partition by tuple() settings min_bytes_for_wide_part = 0; show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); optimize table ttl_00933_1 final; @@ -60,7 +63,7 @@ select * from ttl_00933_1; -- const Date TTL positive drop table if exists ttl_00933_1; -create table ttl_00933_1 (b Int, a Int ttl today()-1) engine = MergeTree order by tuple() partition by tuple(); +create table ttl_00933_1 (b Int, a Int ttl today()-1) engine = MergeTree order by tuple() partition by tuple() settings min_bytes_for_wide_part = 0; show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); optimize table ttl_00933_1 final; @@ -68,7 +71,7 @@ select * from ttl_00933_1; -- const Date TTL negative drop table if exists ttl_00933_1; -create table ttl_00933_1 (b Int, a Int ttl today()+1) engine = MergeTree order by tuple() partition by tuple(); +create table ttl_00933_1 (b Int, a Int ttl today()+1) engine = MergeTree order by tuple() partition by tuple() settings min_bytes_for_wide_part = 0; show create table ttl_00933_1; insert into ttl_00933_1 values (1, 1); optimize table ttl_00933_1 final; From b3124d2cb0d8bae72f68d027d11cb40f2e6e4315 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 10 Sep 2020 23:51:30 +0300 Subject: [PATCH 052/390] adding test outputs --- cmake_files_header.md | 12 +-- cmake_flags_and_output.md | 173 ++++++++++++++++++++++++++++++++++++ cmake_flags_md_generator.sh | 24 +++++ 3 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 cmake_flags_and_output.md mode change 100644 => 100755 cmake_flags_md_generator.sh diff --git a/cmake_files_header.md b/cmake_files_header.md index bb613edaf1b..17f006a7760 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -22,12 +22,12 @@ If the option's purpose can't be guessed by its name, or the purpose guess may b the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). The comment is parsed into a separate column (see below). -Even better: +Even better (default off value is omitted): ``` # Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). # see tests/CMakeLists.txt for implementation detail. -option(ENABLE_GTEST_TESTS OFF) +option(ENABLE_GTEST_TESTS) ``` ### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. @@ -113,7 +113,7 @@ Better: ``` # https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE OFF) +option (USE_INCLUDE_WHAT_YOU_USE) ``` ### Prefer consistent default values. @@ -129,9 +129,3 @@ Prefer the `ON/OFF` values, if possible. | Name | Description | Default value | Comment | |------|-------------|---------------|---------| -| \#0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | -| \#1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | -| \#2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | -| \#N | … | … | … | … | … | - - diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md new file mode 100644 index 00000000000..8bfe00c5408 --- /dev/null +++ b/cmake_flags_and_output.md @@ -0,0 +1,173 @@ +# CMake flags + +## Developer's guide for adding new CMake options + +### Don't be obvious. Be informative. + +Bad: +``` +option (ENABLE_TESTS "Enables testing" OFF) +``` + +This description is quite useless as is neither gives the viewer any additional information nor explains the option +purpose. If the option's name is quite self-descriptive, prefer the empty description. + +Better: + +``` +option(ENABLE_TESTS OFF) +``` + +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above +the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). +The comment is parsed into a separate column (see below). + +Even better (default off value is omitted): + +``` +# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). +# see tests/CMakeLists.txt for implementation detail. +option(ENABLE_GTEST_TESTS) +``` + +### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. + +Suppose you have an option that may strip debug symbols from the ClickHouse's part. +This can speed up the linking process, but produces a binary that cannot be debugged. +In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. +Also, such options should be disabled if applies. + +Bad: +``` +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions. + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() + +``` +Better: + +``` +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions." + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + message(WARNING "Not generating debugger info for ClickHouse functions") + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() +``` + +### In the option's description, explain WHAT the option does rather than WHY it does something. + +The WHY explanation should be placed in the comment. +You may find that the option's name is self-descriptive. + +Bad: + +``` +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better: + +``` +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +### Don't assume other developers know as much as you do. + +In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to +the tool's docs. It won't take much of your time. + +Bad: + +``` +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better (combined with the above hint): + +``` +# https://clang.llvm.org/docs/ThinLTO.html +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +Other example, bad: + +``` +option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) +``` + +Better: + +``` +# https://github.com/include-what-you-use/include-what-you-use +option (USE_INCLUDE_WHAT_YOU_USE) +``` + +### Prefer consistent default values. + +CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. +Prefer the `ON/OFF` values, if possible. + + +## List of CMake flags + +* This list is auto-generated by [this bash script](bash.sh). +* The flag name is a link to its position in the code. + +| Name | Description | Default value | Comment | +|------|-------------|---------------|---------| +| ENABLE_CLANG_TIDY | OFF | Use 'clang-tidy' static analyzer if present | | + +| USE_INTERNAL_ | ${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED} | | | + +| FUZZER | | Enable fuzzer: libfuzzer | | + +| PARALLEL_COMPILE_JOBS | | Define the maximum number of concurrent compilation jobs" " | | + +| PARALLEL_LINK_JOBS | | Define the maximum number of concurrent link jobs" " | | + +| SANITIZE | | Enable sanitizer: address, memory, thread, undefined" " | | + +| LINKER_NAME | | Linker name or full path | | + +| WEVERYTHING | ON | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | + +| ENABLE_CLICKHOUSE_ALL | ON | Enable all tools | | + +| ENABLE_CLICKHOUSE_SERVER | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-server | | + +| ENABLE_CLICKHOUSE_CLIENT | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-client | | + +| ENABLE_CLICKHOUSE_LOCAL | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-local | | + +| ENABLE_CLICKHOUSE_BENCHMARK | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-benchmark | | + +| ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-extract-from-config | | + +| ENABLE_CLICKHOUSE_COMPRESSOR | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-compressor | | + +| ENABLE_CLICKHOUSE_COPIER | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-copier | | + +| ENABLE_CLICKHOUSE_FORMAT | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-format | | + +| ENABLE_CLICKHOUSE_OBFUSCATOR | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-obfuscator | | + +| ENABLE_CLICKHOUSE_ODBC_BRIDGE | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-odbc-bridge | | + +| ENABLE_CLICKHOUSE_INSTALL | OFF | Enable clickhouse-install | | + +| ENABLE_CLICKHOUSE_INSTALL | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-install | | + diff --git a/cmake_flags_md_generator.sh b/cmake_flags_md_generator.sh old mode 100644 new mode 100755 index e72f7660e6d..390840039af --- a/cmake_flags_md_generator.sh +++ b/cmake_flags_md_generator.sh @@ -1 +1,25 @@ #!/bin/bash/ + +output_file_name="cmake_flags_and_output.md" + +regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| \3 | \7 | \5 | \1 |\n/mg;t;d' + +rm -fr ${output_file_name} +touch ${output_file_name} +cat cmake_files_header.md >> ${output_file_name} + +process() { + for i in "$1"/*.cmake "$1"/CMakeLists.txt;do + if [ -d "$i" ];then + process "$i" + elif [ -f "$i" ]; then + echo "Processing $i" + + cat $i | sed -E "${regex}" >> ${output_file_name} + fi + done +} + +for base_folder in ./base ./cmake ./programs ./src; do + process $base_folder +done From 5dbfb3771be7d259175d04df7fac3b9e09974364 Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 11 Sep 2020 00:22:02 +0300 Subject: [PATCH 053/390] updating links --- cmake_files_header.md | 4 +-- cmake_flags_and_output.md | 67 +++++++++++++------------------------ cmake_flags_md_generator.sh | 9 +++-- 3 files changed, 31 insertions(+), 49 deletions(-) diff --git a/cmake_files_header.md b/cmake_files_header.md index 17f006a7760..00a8b327e4c 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -127,5 +127,5 @@ Prefer the `ON/OFF` values, if possible. * This list is auto-generated by [this bash script](bash.sh). * The flag name is a link to its position in the code. -| Name | Description | Default value | Comment | -|------|-------------|---------------|---------| +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 8bfe00c5408..97b8049121f 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -127,47 +127,26 @@ Prefer the `ON/OFF` values, if possible. * This list is auto-generated by [this bash script](bash.sh). * The flag name is a link to its position in the code. -| Name | Description | Default value | Comment | -|------|-------------|---------------|---------| -| ENABLE_CLANG_TIDY | OFF | Use 'clang-tidy' static analyzer if present | | - -| USE_INTERNAL_ | ${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED} | | | - -| FUZZER | | Enable fuzzer: libfuzzer | | - -| PARALLEL_COMPILE_JOBS | | Define the maximum number of concurrent compilation jobs" " | | - -| PARALLEL_LINK_JOBS | | Define the maximum number of concurrent link jobs" " | | - -| SANITIZE | | Enable sanitizer: address, memory, thread, undefined" " | | - -| LINKER_NAME | | Linker name or full path | | - -| WEVERYTHING | ON | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | - -| ENABLE_CLICKHOUSE_ALL | ON | Enable all tools | | - -| ENABLE_CLICKHOUSE_SERVER | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-server | | - -| ENABLE_CLICKHOUSE_CLIENT | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-client | | - -| ENABLE_CLICKHOUSE_LOCAL | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-local | | - -| ENABLE_CLICKHOUSE_BENCHMARK | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-benchmark | | - -| ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-extract-from-config | | - -| ENABLE_CLICKHOUSE_COMPRESSOR | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-compressor | | - -| ENABLE_CLICKHOUSE_COPIER | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-copier | | - -| ENABLE_CLICKHOUSE_FORMAT | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-format | | - -| ENABLE_CLICKHOUSE_OBFUSCATOR | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-obfuscator | | - -| ENABLE_CLICKHOUSE_ODBC_BRIDGE | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-odbc-bridge | | - -| ENABLE_CLICKHOUSE_INSTALL | OFF | Enable clickhouse-install | | - -| ENABLE_CLICKHOUSE_INSTALL | ${ENABLE_CLICKHOUSE_ALL} | Enable clickhouse-install | | - +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| (`ENABLE_CLANG_TIDY`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake] | `OFF` | Use 'clang-tidy' static analyzer if present | | +| (`USE_INTERNAL_`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake] | `${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED}` | | | +| (`FUZZER`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake] | `` | Enable fuzzer: libfuzzer | | +| (`PARALLEL_COMPILE_JOBS`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake] | `` | Define the maximum number of concurrent compilation jobs" " | | +| (`PARALLEL_LINK_JOBS`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake] | `` | Define the maximum number of concurrent link jobs" " | | +| (`SANITIZE`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake] | `` | Enable sanitizer: address, memory, thread, undefined" " | | +| (`LINKER_NAME`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake] | `` | Linker name or full path | | +| (`WEVERYTHING`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake] | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | +| (`ENABLE_CLICKHOUSE_ALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `ON` | Enable all tools | | +| (`ENABLE_CLICKHOUSE_SERVER`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-server | | +| (`ENABLE_CLICKHOUSE_CLIENT`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-client | | +| (`ENABLE_CLICKHOUSE_LOCAL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-local | | +| (`ENABLE_CLICKHOUSE_BENCHMARK`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-benchmark | | +| (`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-extract-from-config | | +| (`ENABLE_CLICKHOUSE_COMPRESSOR`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-compressor | | +| (`ENABLE_CLICKHOUSE_COPIER`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-copier | | +| (`ENABLE_CLICKHOUSE_FORMAT`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-format | | +| (`ENABLE_CLICKHOUSE_OBFUSCATOR`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-obfuscator | | +| (`ENABLE_CLICKHOUSE_ODBC_BRIDGE`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-odbc-bridge | | +| (`ENABLE_CLICKHOUSE_INSTALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `OFF` | Enable clickhouse-install | | +| (`ENABLE_CLICKHOUSE_INSTALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-install | | diff --git a/cmake_flags_md_generator.sh b/cmake_flags_md_generator.sh index 390840039af..6796355222d 100755 --- a/cmake_flags_md_generator.sh +++ b/cmake_flags_md_generator.sh @@ -1,8 +1,7 @@ #!/bin/bash/ output_file_name="cmake_flags_and_output.md" - -regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| \3 | \7 | \5 | \1 |\n/mg;t;d' +ch_master_url="http:\/\/github.com\/clickhouse\/clickhouse\/blob\/master\/" rm -fr ${output_file_name} touch ${output_file_name} @@ -14,8 +13,12 @@ process() { process "$i" elif [ -f "$i" ]; then echo "Processing $i" + subd_name=${i//\//\\/} + subd_name=${subd_name//\./\\\.} + subd_name=${subd_name:2} + regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| (`\3`)['$ch_master_url${subd_name:2}'] | `\7` | \5 | \1 |/mg;t;d' - cat $i | sed -E "${regex}" >> ${output_file_name} + cat $i | sed -E "$regex" >> ${output_file_name} fi done } From 3319933dc9ecb463d65edcc724576192820d5b0a Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 11 Sep 2020 00:48:56 +0300 Subject: [PATCH 054/390] fix: subsitution --- cmake_files_header.md | 22 +++--- cmake_flags_and_output.md | 151 ++++++++++++++++++++++++++++-------- cmake_flags_md_generator.sh | 28 ++++--- 3 files changed, 148 insertions(+), 53 deletions(-) diff --git a/cmake_files_header.md b/cmake_files_header.md index 00a8b327e4c..e1fa8ad29fe 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -5,7 +5,7 @@ ### Don't be obvious. Be informative. Bad: -``` +```cmake option (ENABLE_TESTS "Enables testing" OFF) ``` @@ -14,7 +14,7 @@ purpose. If the option's name is quite self-descriptive, prefer the empty descri Better: -``` +```cmake option(ENABLE_TESTS OFF) ``` @@ -24,7 +24,7 @@ The comment is parsed into a separate column (see below). Even better (default off value is omitted): -``` +```cmake # Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). # see tests/CMakeLists.txt for implementation detail. option(ENABLE_GTEST_TESTS) @@ -38,7 +38,7 @@ In that case, prefer explicitly raising a warning telling the developer that he Also, such options should be disabled if applies. Bad: -``` +```cmake option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions. ${STRIP_DSF_DEFAULT}) @@ -50,7 +50,7 @@ endif() ``` Better: -``` +```cmake # Provides faster linking and lower binary size. # Tradeoff is the inability to debug some source files with e.g. gdb # (empty stack frames and no local variables)." @@ -71,13 +71,13 @@ You may find that the option's name is self-descriptive. Bad: -``` +```cmake option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) ``` Better: -``` +```cmake # Only applicable for clang. # Turned off when building with tests or sanitizers. option(ENABLE_THINLTO ON). @@ -90,13 +90,13 @@ the tool's docs. It won't take much of your time. Bad: -``` +```cmake option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) ``` Better (combined with the above hint): -``` +```cmake # https://clang.llvm.org/docs/ThinLTO.html # Only applicable for clang. # Turned off when building with tests or sanitizers. @@ -105,13 +105,13 @@ option(ENABLE_THINLTO ON). Other example, bad: -``` +```cmake option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) ``` Better: -``` +```cmake # https://github.com/include-what-you-use/include-what-you-use option (USE_INCLUDE_WHAT_YOU_USE) ``` diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 97b8049121f..65ad40dc916 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -5,7 +5,7 @@ ### Don't be obvious. Be informative. Bad: -``` +```cmake option (ENABLE_TESTS "Enables testing" OFF) ``` @@ -14,7 +14,7 @@ purpose. If the option's name is quite self-descriptive, prefer the empty descri Better: -``` +```cmake option(ENABLE_TESTS OFF) ``` @@ -24,7 +24,7 @@ The comment is parsed into a separate column (see below). Even better (default off value is omitted): -``` +```cmake # Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). # see tests/CMakeLists.txt for implementation detail. option(ENABLE_GTEST_TESTS) @@ -38,7 +38,7 @@ In that case, prefer explicitly raising a warning telling the developer that he Also, such options should be disabled if applies. Bad: -``` +```cmake option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions. ${STRIP_DSF_DEFAULT}) @@ -50,7 +50,7 @@ endif() ``` Better: -``` +```cmake # Provides faster linking and lower binary size. # Tradeoff is the inability to debug some source files with e.g. gdb # (empty stack frames and no local variables)." @@ -71,13 +71,13 @@ You may find that the option's name is self-descriptive. Bad: -``` +```cmake option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) ``` Better: -``` +```cmake # Only applicable for clang. # Turned off when building with tests or sanitizers. option(ENABLE_THINLTO ON). @@ -90,13 +90,13 @@ the tool's docs. It won't take much of your time. Bad: -``` +```cmake option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) ``` Better (combined with the above hint): -``` +```cmake # https://clang.llvm.org/docs/ThinLTO.html # Only applicable for clang. # Turned off when building with tests or sanitizers. @@ -105,13 +105,13 @@ option(ENABLE_THINLTO ON). Other example, bad: -``` +```cmake option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) ``` Better: -``` +```cmake # https://github.com/include-what-you-use/include-what-you-use option (USE_INCLUDE_WHAT_YOU_USE) ``` @@ -124,29 +124,114 @@ Prefer the `ON/OFF` values, if possible. ## List of CMake flags -* This list is auto-generated by [this bash script](bash.sh). +* This list is auto-generated by [this bash script](cmake_flags_md_generator.sh). * The flag name is a link to its position in the code. | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| (`ENABLE_CLANG_TIDY`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake] | `OFF` | Use 'clang-tidy' static analyzer if present | | -| (`USE_INTERNAL_`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake] | `${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED}` | | | -| (`FUZZER`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake] | `` | Enable fuzzer: libfuzzer | | -| (`PARALLEL_COMPILE_JOBS`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake] | `` | Define the maximum number of concurrent compilation jobs" " | | -| (`PARALLEL_LINK_JOBS`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake] | `` | Define the maximum number of concurrent link jobs" " | | -| (`SANITIZE`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake] | `` | Enable sanitizer: address, memory, thread, undefined" " | | -| (`LINKER_NAME`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake] | `` | Linker name or full path | | -| (`WEVERYTHING`)[http://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake] | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | -| (`ENABLE_CLICKHOUSE_ALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `ON` | Enable all tools | | -| (`ENABLE_CLICKHOUSE_SERVER`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-server | | -| (`ENABLE_CLICKHOUSE_CLIENT`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-client | | -| (`ENABLE_CLICKHOUSE_LOCAL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-local | | -| (`ENABLE_CLICKHOUSE_BENCHMARK`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-benchmark | | -| (`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-extract-from-config | | -| (`ENABLE_CLICKHOUSE_COMPRESSOR`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-compressor | | -| (`ENABLE_CLICKHOUSE_COPIER`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-copier | | -| (`ENABLE_CLICKHOUSE_FORMAT`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-format | | -| (`ENABLE_CLICKHOUSE_OBFUSCATOR`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-obfuscator | | -| (`ENABLE_CLICKHOUSE_ODBC_BRIDGE`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-odbc-bridge | | -| (`ENABLE_CLICKHOUSE_INSTALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `OFF` | Enable clickhouse-install | | -| (`ENABLE_CLICKHOUSE_INSTALL`)[http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt] | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-install | | +| [`ENABLE_IPO`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `` | Enable full link time optimization | | +| [`USE_STATIC_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Set to FALSE to use shared libraries | | +| [`MAKE_STATIC_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to make shared libraries | | +| [`SPLIT_SHARED_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | DEV ONLY. Keep all internal libs as separate .so for faster linking | | +| [`CLICKHOUSE_SPLIT_BINARY`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... ) | | +| [`ENABLE_FUZZING`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Enables fuzzing instrumentation | | +| [`ENABLE_TESTS`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enables tests | | +| [`GLIBC_COMPATIBILITY`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY. | | +| [`ADD_GDB_INDEX_FOR_GOLD`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| [`COMPILER_PIPE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`ARCH_NATIVE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Enable -march=native compiler flag | | +| [`WITH_COVERAGE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Build with coverage. | | +| [`ENABLE_THINLTO`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers. | | +| [`ENABLE_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable all libraries (Global default switch) | | +| [`UNBUNDLED`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages. | | +| [`WERROR`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Enable -Werror compiler option | | +| [`WERROR`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable -Werror compiler option | | +| [`USE_INCLUDE_WHAT_YOU_USE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Use 'include-what-you-use' tool | | +| [`ENABLE_CLANG_TIDY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake) | `OFF` | Use 'clang-tidy' static analyzer if present | | +| [`USE_INTERNAL_`](http://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake) | `${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED}` | | | +| [`FUZZER`](http://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake) | `` | Enable fuzzer: libfuzzer | | +| [`PARALLEL_COMPILE_JOBS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake) | `` | Define the maximum number of concurrent compilation jobs" " | | +| [`PARALLEL_LINK_JOBS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake) | `` | Define the maximum number of concurrent link jobs" " | | +| [`SANITIZE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake) | `` | Enable sanitizer: address, memory, thread, undefined" " | | +| [`LINKER_NAME`](http://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake) | `` | Linker name or full path | | +| [`WEVERYTHING`](http://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | +| [`ENABLE_AMQPCPP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake) | `${ENABLE_LIBRARIES}` | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake) | `${ENABLE_LIBRARIES}` | Enable Avro | | +| [`ENABLE_BASE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake) | `64 "Enable base64" ${ENABLE_LIBRARIES}` | | | +| [`ENABLE_BROTLI`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `${ENABLE_LIBRARIES}` | Enable brotli | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | +| [`ENABLE_CAPNP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake) | `${ENABLE_LIBRARIES}` | Enable Cap'n Proto | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system capnproto library instead of bundled | | +| [`ENABLE_CASSANDRA`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake) | `${ENABLE_LIBRARIES}` | Enable Cassandra | | +| [`ENABLE_CCACHE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake) | `${ENABLE_CCACHE_BY_DEFAULT}` | Speedup re-compilations using ccache | | +| [`ENABLE_CURL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake) | `${ENABLE_LIBRARIES}` | Enable curl | | +| [`USE_INTERNAL_CURL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake) | `${NOT_UNBUNDLED}` | Use internal curl library | | +| [`USE_LIBCXX`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake) | `${NOT_UNBUNDLED}` | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake) | `${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}` | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | +| [`ENABLE_FASTOPS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake) | `${ENABLE_LIBRARIES}` | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | +| [`ENABLE_GPERF`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake) | `${ENABLE_LIBRARIES}` | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake) | `${ENABLE_LIBRARIES}` | Use gRPC | | +| [`ENABLE_GTEST_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake) | `${ENABLE_LIBRARIES}` | Enable gtest library | | +| [`USE_INTERNAL_GTEST_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system Google Test instead of bundled | | +| [`ENABLE_H`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake) | `3 "Enable H3" ${ENABLE_LIBRARIES}` | | | +| [`ENABLE_HDFS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake) | `${ENABLE_LIBRARIES}` | Enable HDFS | | +| [`USE_INTERNAL_HDFS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake) | `3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk` | | | +| [`ENABLE_ICU`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `${ENABLE_LIBRARIES}` | Enable ICU | | +| [`ENABLE_ICU`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `0` | Enable ICU | | +| [`USE_INTERNAL_ICU_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system ICU library instead of bundled | | +| [`ENABLE_LDAP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake) | `${ENABLE_LIBRARIES}` | Enable LDAP | | +| [`USE_INTERNAL_LDAP_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system *LDAP library instead of bundled | | +| [`ENABLE_GSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `${ENABLE_LIBRARIES}` | Enable gsasl library | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBXML`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake) | `2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ${NOT_UNBUNDLED}` | | | +| [`ENABLE_EMBEDDED_COMPILER`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `${ENABLE_LIBRARIES}` | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`USE_INTERNAL_LLVM_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `${NOT_UNBUNDLED}` | Use bundled or system LLVM library. | | +| [`LLVM_HAS_RTTI`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `ON` | Enable if LLVM was build with RTTI enabled | | +| [`ENABLE_MSGPACK`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake) | `${ENABLE_LIBRARIES}` | Enable msgpack library | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system msgpack library instead of bundled | | +| [`ENABLE_MYSQL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `${ENABLE_LIBRARIES}` | Enable MySQL | | +| [`ENABLE_MYSQL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `FALSE` | Enable MySQL | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system mysqlclient library instead of bundled | | +| [`ENABLE_ODBC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake) | `${ENABLE_LIBRARIES}` | Enable ODBC library | | +| [`USE_INTERNAL_ODBC_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake) | `${NOT_UNBUNDLED}` | Use internal ODBC library | | +| [`ENABLE_OPENCL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake) | `${ENABLE_LIBRARIES}` | Enable OpenCL support | | +| [`ENABLE_ORC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake) | `${ENABLE_LIBRARIES}` | Enable ORC | | +| [`USE_INTERNAL_ORC_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake) | `"Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk` | | | +| [`ENABLE_PARQUET`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake) | `${ENABLE_LIBRARIES}` | Enable parquet | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system parquet library instead of bundled | | +| [`USE_INTERNAL_POCO_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake) | `ON` | Use internal Poco library | | +| [`ENABLE_PROTOBUF`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake) | `${ENABLE_LIBRARIES}` | Enable protobuf | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system protobuf instead of bundled | | +| [`ENABLE_RAPIDJSON`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake) | `${ENABLE_LIBRARIES}` | Use rapidjson | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system rapidjson library instead of bundled | | +| [`ENABLE_RDKAFKA`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake) | `${ENABLE_LIBRARIES}` | Enable kafka | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system librdkafka instead of the bundled | | +| [`USE_INTERNAL_RE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake) | `2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ${NOT_UNBUNDLED}` | | | +| [`ENABLE_S`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake) | `3 "Enable S3" ${ENABLE_LIBRARIES}` | | | +| [`USE_INTERNAL_AWS_S`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake) | `3_LIBRARY "Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk` | | | +| [`USE_SENTRY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake) | `${ENABLE_LIBRARIES}` | Use Sentry | | +| [`USE_SIMDJSON`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake) | `${ENABLE_LIBRARIES}` | Use simdjson | | +| [`USE_SNAPPY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake) | `${ENABLE_LIBRARIES}` | Enable support of snappy library | | +| [`USE_INTERNAL_SNAPPY_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system snappy library instead of bundled | | +| [`ENABLE_SSL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake) | `${ENABLE_LIBRARIES}` | Enable ssl | | +| [`USE_INTERNAL_SSL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system *ssl library instead of bundled | | +| [`ENABLE_STATS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake) | `${ENABLE_LIBRARIES}` | Enalbe StatsLib library | | +| [`USE_UNWIND`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake) | `${ENABLE_LIBRARIES}` | Enable libunwind (better stacktraces) | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system zstd library instead of bundled | | +| [`ENABLE_CLICKHOUSE_ALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `ON` | Enable all tools | | +| [`ENABLE_CLICKHOUSE_SERVER`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-server | | +| [`ENABLE_CLICKHOUSE_CLIENT`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-client | | +| [`ENABLE_CLICKHOUSE_LOCAL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-local | | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-benchmark | | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-extract-from-config | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-compressor | | +| [`ENABLE_CLICKHOUSE_COPIER`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-copier | | +| [`ENABLE_CLICKHOUSE_FORMAT`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-format | | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-obfuscator | | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-odbc-bridge | | +| [`ENABLE_CLICKHOUSE_INSTALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `OFF` | Enable clickhouse-install | | +| [`ENABLE_CLICKHOUSE_INSTALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-install | | +| [`ENABLE_MULTITARGET_CODE`](http://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt) | `ON` | | | diff --git a/cmake_flags_md_generator.sh b/cmake_flags_md_generator.sh index 6796355222d..17ccc8a1bba 100755 --- a/cmake_flags_md_generator.sh +++ b/cmake_flags_md_generator.sh @@ -1,5 +1,7 @@ #!/bin/bash/ +# https://regex101.com/r/R6iogw/7 + output_file_name="cmake_flags_and_output.md" ch_master_url="http:\/\/github.com\/clickhouse\/clickhouse\/blob\/master\/" @@ -9,20 +11,28 @@ cat cmake_files_header.md >> ${output_file_name} process() { for i in "$1"/*.cmake "$1"/CMakeLists.txt;do - if [ -d "$i" ];then - process "$i" - elif [ -f "$i" ]; then - echo "Processing $i" - subd_name=${i//\//\\/} - subd_name=${subd_name//\./\\\.} - subd_name=${subd_name:2} - regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| (`\3`)['$ch_master_url${subd_name:2}'] | `\7` | \5 | \1 |/mg;t;d' + echo "Processing $i" + subd_name=${i//\//\\/} + subd_name=${subd_name//\./\\\.} + subd_name=${subd_name:2} + regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| [`\3`]('$ch_master_url${subd_name:2}') | `\7` | \5 | \1 |/mg;t;d' + if [ -f $i ]; then cat $i | sed -E "$regex" >> ${output_file_name} fi done + + if [ "$2" = true ] ; then + for d in "$1"/*;do + if [ -d "$d" ];then + process $d + fi + done + fi } +process ./ false + for base_folder in ./base ./cmake ./programs ./src; do - process $base_folder + process $base_folder true done From 8a201a28c04e06f1a2ebb03d51c0e1d8e983680a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 11 Sep 2020 03:14:35 +0300 Subject: [PATCH 055/390] remove skip list for tests with polymorphic parts --- tests/clickhouse-test | 9 --------- tests/queries/skip_list.json | 27 --------------------------- 2 files changed, 36 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 6bfad37d8ad..a3bed189d55 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -506,15 +506,6 @@ def collect_build_flags(client): else: raise Exception("Cannot get inforamtion about build from server errorcode {}, stderr {}".format(clickhouse_proc.returncode, stderr)) - clickhouse_proc = Popen(shlex.split(client), stdin=PIPE, stdout=PIPE, stderr=PIPE) - (stdout, stderr) = clickhouse_proc.communicate("SELECT value FROM system.merge_tree_settings WHERE name = 'min_bytes_for_wide_part'") - - if clickhouse_proc.returncode == 0: - if '10485760' in stdout: - result.append(BuildFlags.POLYMORPHIC_PARTS) - else: - raise Exception("Cannot get inforamtion about build from server errorcode {}, stderr {}".format(clickhouse_proc.returncode, stderr)) - return result diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index adfc5f0e582..535f2757e43 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -127,32 +127,5 @@ "01130_in_memory_parts_partitons", "01225_show_create_table_from_dictionary", "01224_no_superfluous_dict_reload" - ], - "polymorphic-parts": [ - /// These tests fail with compact parts, because they - /// check some implementation defined things - /// like checksums, computed granularity, ProfileEvents, etc. - "avx", - "01045_order_by_pk_special_storages", - "01042_check_query_and_last_granule_size", - "00961_checksums_in_system_parts_columns_table", - "00933_test_fix_extra_seek_on_compressed_cache", - "00926_adaptive_index_granularity_collapsing_merge_tree", - "00926_adaptive_index_granularity_merge_tree", - "00926_adaptive_index_granularity_replacing_merge_tree", - "00926_adaptive_index_granularity_versioned_collapsing_merge_tree", - "00804_test_delta_codec_compression", - "00731_long_merge_tree_select_opened_files", - "00653_verification_monotonic_data_load", - "00484_preferred_max_column_in_block_size_bytes", - "00446_clear_column_in_partition_zookeeper", - "00443_preferred_block_size_bytes", - "00160_merge_and_index_in_in", - "01055_compact_parts", - "01039_mergetree_exec_time", - "00933_ttl_simple", /// Maybe it's worth to fix it - "00753_system_columns_and_system_tables", - "01343_min_bytes_to_use_mmap_io", - "01344_min_bytes_to_use_mmap_io_index" ] } From 6047df2c37c6afea1afabd08cc4406c36f9a62e0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2020 06:18:36 +0000 Subject: [PATCH 056/390] Bump numpy from 1.19.1 to 1.19.2 in /docs/tools Bumps [numpy](https://github.com/numpy/numpy) from 1.19.1 to 1.19.2. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.19.1...v1.19.2) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 9e916489ea4..a3949892829 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -22,7 +22,7 @@ mkdocs-macros-plugin==0.4.9 nltk==3.5 nose==1.3.7 protobuf==3.13.0 -numpy==1.19.1 +numpy==1.19.2 Pygments==2.5.2 pymdown-extensions==8.0 python-slugify==4.0.1 From 7ff7ee6aac2d234625780eeb98e4cfbb7b88c5f8 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Fri, 11 Sep 2020 15:37:14 +0800 Subject: [PATCH 057/390] ISSUES-4006 try fix integration test --- .../test_materialize_mysql_database/materialize_with_ddl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index a953202bff0..869c2e88c96 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -105,6 +105,8 @@ def materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, mysql_ mysql_node.query("INSERT INTO test_database.test_table_1 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializeMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) + assert "test_database" in clickhouse_node.query("SHOW DATABASES") + check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") check_query(clickhouse_node, "SELECT * FROM test_database.test_table_1 ORDER BY key FORMAT TSV", "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" From dd867b787f0de6d6d7dca46a6bcf451990ceed6d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 3 Sep 2020 01:35:47 +0300 Subject: [PATCH 058/390] Allow parallel execution of distributed DDL Add distributed_ddl.pool_size to control maximum parallel to handle distributed DDL. Also: - convert Exception constructors to fmt-like - use sleepFor* over std::this_thread::sleep_for() --- programs/server/Server.cpp | 5 +- programs/server/config.xml | 3 + src/Interpreters/DDLWorker.cpp | 259 +++++++++++++++++---------------- src/Interpreters/DDLWorker.h | 31 ++-- 4 files changed, 162 insertions(+), 136 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f24ba444203..e4fd351f091 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -708,7 +708,10 @@ int Server::main(const std::vector & /*args*/) { /// DDL worker should be started after all tables were loaded String ddl_zookeeper_path = config().getString("distributed_ddl.path", "/clickhouse/task_queue/ddl/"); - global_context->setDDLWorker(std::make_unique(ddl_zookeeper_path, *global_context, &config(), "distributed_ddl")); + int pool_size = config().getInt("distributed_ddl.pool_size", 1); + if (pool_size < 1) + throw Exception("distributed_ddl.pool_size should be greater then 0", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + global_context->setDDLWorker(std::make_unique(pool_size, ddl_zookeeper_path, *global_context, &config(), "distributed_ddl")); } std::unique_ptr dns_cache_updater; diff --git a/programs/server/config.xml b/programs/server/config.xml index af01e880dc2..d13978f9ee8 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -615,6 +615,9 @@ + + + diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index b9b52e2f3fe..526f15d921f 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -38,10 +37,11 @@ #include #include #include +#include #include +#include #include #include -#include namespace DB @@ -144,7 +144,7 @@ struct DDLLogEntry rb >> "version: " >> version >> "\n"; if (version != CURRENT_VERSION) - throw Exception("Unknown DDLLogEntry format version: " + DB::toString(version), ErrorCodes::UNKNOWN_FORMAT_VERSION); + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}", version); Strings host_id_strings; rb >> "query: " >> escape >> query >> "\n"; @@ -308,9 +308,14 @@ static bool isSupportedAlterType(int type) } -DDLWorker::DDLWorker(const std::string & zk_root_dir, Context & context_, const Poco::Util::AbstractConfiguration * config, const String & prefix) - : context(context_), log(&Poco::Logger::get("DDLWorker")) +DDLWorker::DDLWorker(int pool_size_, const std::string & zk_root_dir, Context & context_, const Poco::Util::AbstractConfiguration * config, const String & prefix) + : context(context_) + , log(&Poco::Logger::get("DDLWorker")) + , pool_size(pool_size_) + , worker_pool(pool_size_) { + last_tasks.reserve(pool_size); + queue_dir = zk_root_dir; if (queue_dir.back() == '/') queue_dir.resize(queue_dir.size() - 1); @@ -343,6 +348,7 @@ DDLWorker::~DDLWorker() stop_flag = true; queue_updated_event->set(); cleanup_event->set(); + worker_pool.wait(); main_thread.join(); cleanup_thread.join(); } @@ -364,8 +370,27 @@ DDLWorker::ZooKeeperPtr DDLWorker::getAndSetZooKeeper() return current_zookeeper; } +void DDLWorker::recoverZooKeeper() +{ + LOG_DEBUG(log, "Recovering ZooKeeper session after: {}", getCurrentExceptionMessage(false)); -bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, const ZooKeeperPtr & zookeeper) + while (!stop_flag) + { + try + { + getAndSetZooKeeper(); + break; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + sleepForSeconds(5); + } + } +} + + +DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, const ZooKeeperPtr & zookeeper) { String node_data; String entry_path = queue_dir + "/" + entry_name; @@ -374,7 +399,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, { /// It is Ok that node could be deleted just now. It means that there are no current host in node's host list. out_reason = "The task was deleted"; - return false; + return {}; } auto task = std::make_unique(); @@ -405,7 +430,7 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, } out_reason = "Incorrect task format"; - return false; + return {}; } bool host_in_hostlist = false; @@ -433,12 +458,13 @@ bool DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, } } - if (host_in_hostlist) - current_task = std::move(task); - else + if (!host_in_hostlist) + { out_reason = "There is no a local address in host list"; + return {}; + } - return host_in_hostlist; + return task; } @@ -448,10 +474,9 @@ static void filterAndSortQueueNodes(Strings & all_nodes) std::sort(all_nodes.begin(), all_nodes.end()); } - -void DDLWorker::processTasks() +void DDLWorker::scheduleTasks() { - LOG_DEBUG(log, "Processing tasks"); + LOG_DEBUG(log, "Scheduling tasks"); auto zookeeper = tryGetZooKeeper(); Strings queue_nodes = zookeeper->getChildren(queue_dir, nullptr, queue_updated_event); @@ -459,86 +484,60 @@ void DDLWorker::processTasks() if (queue_nodes.empty()) return; - bool server_startup = last_processed_task_name.empty(); + bool server_startup = last_tasks.empty(); auto begin_node = server_startup ? queue_nodes.begin() - : std::upper_bound(queue_nodes.begin(), queue_nodes.end(), last_processed_task_name); + : std::upper_bound(queue_nodes.begin(), queue_nodes.end(), last_tasks.back()); for (auto it = begin_node; it != queue_nodes.end(); ++it) { String entry_name = *it; - if (current_task) + String reason; + auto task = initAndCheckTask(entry_name, reason, zookeeper); + if (!task) { - if (current_task->entry_name == entry_name) - { - LOG_INFO(log, "Trying to process task {} again", entry_name); - } - else - { - LOG_INFO(log, "Task {} was deleted from ZooKeeper before current host committed it", current_task->entry_name); - current_task = nullptr; - } + LOG_DEBUG(log, "Will not execute task {}: {}", entry_name, reason); + saveTask(entry_name); + continue; } - if (!current_task) + bool already_processed = zookeeper->exists(task->entry_path + "/finished/" + task->host_id_str); + if (!server_startup && !task->was_executed && already_processed) { - String reason; - if (!initAndCheckTask(entry_name, reason, zookeeper)) - { - LOG_DEBUG(log, "Will not execute task {}: {}", entry_name, reason); - last_processed_task_name = entry_name; - continue; - } - } - - DDLTask & task = *current_task; - - bool already_processed = zookeeper->exists(task.entry_path + "/finished/" + task.host_id_str); - if (!server_startup && !task.was_executed && already_processed) - { - throw Exception( - "Server expects that DDL task " + task.entry_name + " should be processed, but it was already processed according to ZK", - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Server expects that DDL task {} should be processed, but it was already processed according to ZK", + entry_name); } if (!already_processed) { - try + worker_pool.scheduleOrThrowOnError([this, task_ptr = task.release()]() { - processTask(task, zookeeper); - } - catch (const Coordination::Exception & e) - { - if (server_startup && e.code == Coordination::Error::ZNONODE) - { - LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); - } - else - { - throw; - } - } - catch (...) - { - LOG_WARNING(log, "An error occurred while processing task {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); - throw; - } + enqueueTask(DDLTaskPtr(task_ptr)); + }); } else { - LOG_DEBUG(log, "Task {} ({}) has been already processed", task.entry_name, task.entry.query); + LOG_DEBUG(log, "Task {} ({}) has been already processed", entry_name, task->entry.query); } - last_processed_task_name = task.entry_name; - current_task.reset(); + saveTask(entry_name); if (stop_flag) break; } } +void DDLWorker::saveTask(const String & entry_name) +{ + if (last_tasks.size() == pool_size) + { + last_tasks.erase(last_tasks.begin()); + } + last_tasks.emplace_back(entry_name); +} /// Parses query and resolves cluster and host in cluster void DDLWorker::parseQueryAndResolveHost(DDLTask & task) @@ -559,10 +558,9 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) task.cluster_name = task.query_on_cluster->cluster; task.cluster = context.tryGetCluster(task.cluster_name); if (!task.cluster) - { - throw Exception("DDL task " + task.entry_name + " contains current host " + task.host_id.readableString() - + " in cluster " + task.cluster_name + ", but there are no such cluster here.", ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); - } + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "DDL task {} contains current host {} in cluster {}, but there are no such cluster here.", + task.entry_name, task.host_id.readableString(), task.cluster_name); /// Try to find host from task host list in cluster /// At the first, try find exact match (host name and ports should be literally equal) @@ -583,10 +581,9 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) { if (default_database == address.default_database) { - throw Exception( - "There are two exactly the same ClickHouse instances " + address.readableString() + " in cluster " - + task.cluster_name, - ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "There are two exactly the same ClickHouse instances {} in cluster {}", + address.readableString(), task.cluster_name); } else { @@ -600,9 +597,8 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) auto * query_with_table = dynamic_cast(task.query.get()); if (!query_with_table || query_with_table->database.empty()) { - throw Exception( - "For a distributed DDL on circular replicated cluster its table name must be qualified by database name.", - ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "For a distributed DDL on circular replicated cluster its table name must be qualified by database name."); } if (default_database == query_with_table->database) return; @@ -635,8 +631,9 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) { if (found_via_resolving) { - throw Exception("There are two the same ClickHouse instances in cluster " + task.cluster_name + " : " - + task.address_in_cluster.readableString() + " and " + address.readableString(), ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "There are two the same ClickHouse instances in cluster {} : {} and {}", + task.cluster_name, task.address_in_cluster.readableString(), address.readableString()); } else { @@ -651,8 +648,9 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) if (!found_via_resolving) { - throw Exception("Not found host " + task.host_id.readableString() + " in definition of cluster " + task.cluster_name, - ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "Not found host {} in definition of cluster {}", + task.host_id.readableString(), task.cluster_name); } else { @@ -673,7 +671,7 @@ bool DDLWorker::tryExecuteQuery(const String & query, const DDLTask & task, Exec try { - current_context = std::make_unique(context); + auto current_context = std::make_unique(context); current_context->getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; current_context->setCurrentQueryId(""); // generate random query_id executeQuery(istr, ostr, false, *current_context, {}); @@ -707,8 +705,44 @@ void DDLWorker::attachToThreadGroup() } -void DDLWorker::processTask(DDLTask & task, const ZooKeeperPtr & zookeeper) +void DDLWorker::enqueueTask(DDLTaskPtr task_ptr) { + auto & task = *task_ptr; + + while (!stop_flag) + { + try + { + processTask(task); + return; + } + catch (const Coordination::Exception & e) + { + if (Coordination::isHardwareError(e.code)) + { + recoverZooKeeper(); + } + else if (e.code == Coordination::Error::ZNONODE) + { + LOG_ERROR(log, "ZooKeeper error: {}", getCurrentExceptionMessage(true)); + // TODO: retry? + } + else + { + LOG_ERROR(log, "Unexpected ZooKeeper error: {}.", getCurrentExceptionMessage(true)); + return; + } + } + catch (...) + { + LOG_WARNING(log, "An error occurred while processing task {} ({}) : {}", task.entry_name, task.entry.query, getCurrentExceptionMessage(true)); + } + } +} +void DDLWorker::processTask(DDLTask & task) +{ + auto zookeeper = tryGetZooKeeper(); + LOG_DEBUG(log, "Processing task {} ({})", task.entry_name, task.entry.query); String dummy; @@ -816,16 +850,17 @@ void DDLWorker::checkShardConfig(const String & table, const DDLTask & task, Sto if (storage->supportsReplication() && !config_is_replicated_shard) { - throw Exception("Table " + backQuote(table) + " is replicated, but shard #" + toString(task.host_shard_num + 1) + - " isn't replicated according to its cluster definition." - " Possibly true is forgotten in the cluster config.", - ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "Table {} is replicated, but shard #{} isn't replicated according to its cluster definition. " + "Possibly true is forgotten in the cluster config.", + backQuote(table), task.host_shard_num + 1); } if (!storage->supportsReplication() && config_is_replicated_shard) { - throw Exception("Table " + backQuote(table) + " isn't replicated, but shard #" + toString(task.host_shard_num + 1) + - " is replicated according to its cluster definition", ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION); + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "Table {} isn't replicated, but shard #{} is replicated according to its cluster definition", + backQuote(table), task.host_shard_num + 1); } } @@ -841,7 +876,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( /// If we will develop new replicated storage if (!replicated_storage) - throw Exception("Storage type '" + storage->getName() + "' is not supported by distributed DDL", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Storage type '{}' is not supported by distributed DDL", storage->getName()); /// Generate unique name for shard node, it will be used to execute the query by only single host /// Shard node name has format 'replica_name1,replica_name2,...,replica_nameN' @@ -1118,7 +1153,7 @@ void DDLWorker::runMainThread() attachToThreadGroup(); cleanup_event->set(); - processTasks(); + scheduleTasks(); LOG_DEBUG(log, "Waiting a watch"); queue_updated_event->wait(); @@ -1127,23 +1162,7 @@ void DDLWorker::runMainThread() { if (Coordination::isHardwareError(e.code)) { - LOG_DEBUG(log, "Recovering ZooKeeper session after: {}", getCurrentExceptionMessage(false)); - - while (!stop_flag) - { - try - { - getAndSetZooKeeper(); - break; - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - - using namespace std::chrono_literals; - std::this_thread::sleep_for(5s); - } - } + recoverZooKeeper(); } else if (e.code == Coordination::Error::ZNONODE) { @@ -1260,28 +1279,24 @@ public: size_t num_unfinished_hosts = waiting_hosts.size() - num_hosts_finished; size_t num_active_hosts = current_active_hosts.size(); - std::stringstream msg; - msg << "Watching task " << node_path << " is executing longer than distributed_ddl_task_timeout" - << " (=" << timeout_seconds << ") seconds." - << " There are " << num_unfinished_hosts << " unfinished hosts" - << " (" << num_active_hosts << " of them are currently active)" - << ", they are going to execute the query in background"; - throw Exception(msg.str(), ErrorCodes::TIMEOUT_EXCEEDED); + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, + "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " + "There are {} unfinished hosts ({} of them are currently active), they are going to execute the query in background", + node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); } if (num_hosts_finished != 0 || try_number != 0) { - auto current_sleep_for = std::chrono::milliseconds(std::min(static_cast(1000), 50 * (try_number + 1))); - std::this_thread::sleep_for(current_sleep_for); + sleepForMilliseconds(std::min(1000, 50 * (try_number + 1))); } /// TODO: add shared lock if (!zookeeper->exists(node_path)) { - throw Exception("Cannot provide query execution status. The query's node " + node_path - + " has been deleted by the cleaner since it was finished (or its lifetime is expired)", - ErrorCodes::UNFINISHED); + throw Exception(ErrorCodes::UNFINISHED, + "Cannot provide query execution status. The query's node {} has been deleted by the cleaner since it was finished (or its lifetime is expired)", + node_path); } Strings new_hosts = getNewAndUpdate(getChildrenAllowNoNode(zookeeper, node_path + "/finished")); @@ -1304,7 +1319,7 @@ public: auto [host, port] = Cluster::Address::fromString(host_id); if (status.code != 0 && first_exception == nullptr) - first_exception = std::make_unique("There was an error on [" + host + ":" + toString(port) + "]: " + status.message, status.code); + first_exception = std::make_unique(status.code, "There was an error on [{}:{}]: {}", host, port, status.message); ++num_hosts_finished; diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 544fb3da27d..f6b4dd00684 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -26,6 +26,7 @@ class ASTAlterQuery; class AccessRightsElements; struct DDLLogEntry; struct DDLTask; +using DDLTaskPtr = std::unique_ptr; /// Pushes distributed DDL query to the queue @@ -37,7 +38,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & conte class DDLWorker { public: - DDLWorker(const std::string & zk_root_dir, Context & context_, const Poco::Util::AbstractConfiguration * config, const String & prefix); + DDLWorker(int pool_size_, const std::string & zk_root_dir, Context & context_, const Poco::Util::AbstractConfiguration * config, const String & prefix); ~DDLWorker(); /// Pushes query into DDL queue, returns path to created node @@ -57,14 +58,19 @@ private: ZooKeeperPtr tryGetZooKeeper() const; /// If necessary, creates a new session and caches it. ZooKeeperPtr getAndSetZooKeeper(); + /// ZooKeeper recover loop (while not stopped). + void recoverZooKeeper(); - void processTasks(); + void checkCurrentTasks(); + void scheduleTasks(); + void saveTask(const String & entry_name); /// Reads entry and check that the host belongs to host list of the task - /// Returns true and sets current_task if entry parsed and the check is passed - bool initAndCheckTask(const String & entry_name, String & out_reason, const ZooKeeperPtr & zookeeper); + /// Returns non-empty DDLTaskPtr if entry parsed and the check is passed + DDLTaskPtr initAndCheckTask(const String & entry_name, String & out_reason, const ZooKeeperPtr & zookeeper); - void processTask(DDLTask & task, const ZooKeeperPtr & zookeeper); + void enqueueTask(DDLTaskPtr task); + void processTask(DDLTask & task); /// Check that query should be executed on leader replica only static bool taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, StoragePtr storage); @@ -101,32 +107,31 @@ private: void attachToThreadGroup(); private: - bool is_circular_replicated; + std::atomic is_circular_replicated = false; Context & context; Poco::Logger * log; - std::unique_ptr current_context; std::string host_fqdn; /// current host domain name std::string host_fqdn_id; /// host_name:port std::string queue_dir; /// dir with queue of queries - /// Name of last task that was skipped or successfully executed - std::string last_processed_task_name; - mutable std::mutex zookeeper_mutex; ZooKeeperPtr current_zookeeper; /// Save state of executed task to avoid duplicate execution on ZK error - using DDLTaskPtr = std::unique_ptr; - DDLTaskPtr current_task; + std::vector last_tasks; std::shared_ptr queue_updated_event = std::make_shared(); std::shared_ptr cleanup_event = std::make_shared(); - std::atomic stop_flag{false}; + std::atomic stop_flag = false; ThreadFromGlobalPool main_thread; ThreadFromGlobalPool cleanup_thread; + /// Size of the pool for query execution. + size_t pool_size = 1; + ThreadPool worker_pool; + /// Cleaning starts after new node event is received if the last cleaning wasn't made sooner than N seconds ago Int64 cleanup_delay_period = 60; // minute (in seconds) /// Delete node if its age is greater than that From 9c7f3a9a742fb9b96c176b22b85f4d0a9e8a306c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 12 Sep 2020 02:33:17 +0300 Subject: [PATCH 059/390] Add test_distributed_ddl_parallel --- .../test_distributed_ddl_parallel/__init__.py | 0 .../configs/ddl.xml | 5 ++ .../configs/dict.xml | 26 ++++++ .../configs/remote_servers.xml | 18 ++++ .../test_distributed_ddl_parallel/test.py | 89 +++++++++++++++++++ 5 files changed, 138 insertions(+) create mode 100644 tests/integration/test_distributed_ddl_parallel/__init__.py create mode 100644 tests/integration/test_distributed_ddl_parallel/configs/ddl.xml create mode 100644 tests/integration/test_distributed_ddl_parallel/configs/dict.xml create mode 100644 tests/integration/test_distributed_ddl_parallel/configs/remote_servers.xml create mode 100644 tests/integration/test_distributed_ddl_parallel/test.py diff --git a/tests/integration/test_distributed_ddl_parallel/__init__.py b/tests/integration/test_distributed_ddl_parallel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_ddl_parallel/configs/ddl.xml b/tests/integration/test_distributed_ddl_parallel/configs/ddl.xml new file mode 100644 index 00000000000..b926f99c687 --- /dev/null +++ b/tests/integration/test_distributed_ddl_parallel/configs/ddl.xml @@ -0,0 +1,5 @@ + + + 2 + + diff --git a/tests/integration/test_distributed_ddl_parallel/configs/dict.xml b/tests/integration/test_distributed_ddl_parallel/configs/dict.xml new file mode 100644 index 00000000000..610d55841a0 --- /dev/null +++ b/tests/integration/test_distributed_ddl_parallel/configs/dict.xml @@ -0,0 +1,26 @@ + + + + slow_dict + + + sleep 7 + TabSeparated + + + + + + + + id + + + value + String + + + + 0 + + diff --git a/tests/integration/test_distributed_ddl_parallel/configs/remote_servers.xml b/tests/integration/test_distributed_ddl_parallel/configs/remote_servers.xml new file mode 100644 index 00000000000..8ffa9f024d7 --- /dev/null +++ b/tests/integration/test_distributed_ddl_parallel/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + n1 + 9000 + + + + + n2 + 9000 + + + + + diff --git a/tests/integration/test_distributed_ddl_parallel/test.py b/tests/integration/test_distributed_ddl_parallel/test.py new file mode 100644 index 00000000000..96530b111cb --- /dev/null +++ b/tests/integration/test_distributed_ddl_parallel/test.py @@ -0,0 +1,89 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +from functools import wraps +import threading +import time +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +def add_instance(name): + main_configs=[ + 'configs/ddl.xml', + 'configs/remote_servers.xml', + ] + dictionaries=[ + 'configs/dict.xml', + ] + return cluster.add_instance(name, + main_configs=main_configs, + dictionaries=dictionaries, + with_zookeeper=True) + +initiator = add_instance('initiator') +n1 = add_instance('n1') +n2 = add_instance('n2') + +@pytest.fixture(scope='module', autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +# verifies that functions executes longer then `sec` +def longer_then(sec): + def wrapper(func): + @wraps(func) + def inner(*args, **kwargs): + ts = time.time() + result = func(*args, **kwargs) + te = time.time() + took = te-ts + assert took >= sec + return result + return inner + return wrapper + +# It takes 7 seconds to load slow_dict. +def thread_reload_dictionary(): + initiator.query('SYSTEM RELOAD DICTIONARY ON CLUSTER cluster slow_dict') + +# NOTE: uses inner function to exclude slow start_cluster() from timeout. + +def test_dict_load(): + @pytest.mark.timeout(10) + @longer_then(7) + def inner_test(): + initiator.query('SYSTEM RELOAD DICTIONARY slow_dict') + inner_test() + +def test_all_in_parallel(): + @pytest.mark.timeout(10) + @longer_then(7) + def inner_test(): + threads = [] + for _ in range(2): + threads.append(threading.Thread(target=thread_reload_dictionary)) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + inner_test() + +def test_two_in_parallel_two_queued(): + @pytest.mark.timeout(19) + @longer_then(14) + def inner_test(): + threads = [] + for _ in range(4): + threads.append(threading.Thread(target=thread_reload_dictionary)) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + inner_test() From 7185b9a9452c958eca79e58f77873717b225343d Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sat, 12 Sep 2020 19:59:03 +0300 Subject: [PATCH 060/390] add changes in RU files --- docs/en/operations/settings/settings.md | 2 +- docs/ru/operations/settings/settings.md | 57 ++++++++++++++++++ .../data-types/aggregatefunction.md | 5 ++ docs/ru/sql-reference/data-types/array.md | 5 ++ .../data-types/lowcardinality.md | 59 +++++++++++++++++++ docs/ru/sql-reference/data-types/nullable.md | 5 ++ docs/ru/sql-reference/data-types/tuple.md | 5 ++ .../functions/type-conversion-functions.md | 41 ++++++++++++- 8 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 docs/ru/sql-reference/data-types/lowcardinality.md diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 76fcfa2a616..b1aad4d8e6a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1815,7 +1815,7 @@ Default value: 8192. Turns on or turns off using of single dictionary for the data part. -By default, ClickHouse server monitors the size of dictionaries and if a dictionary overflows then the server starts to write the next one. To prohibit creating several dictionaries set `low_cardinality_use_single_dictionary_for_part = 1`. +By default, the ClickHouse server monitors the size of dictionaries and if a dictionary overflows then the server starts to write the next one. To prohibit creating several dictionaries set `low_cardinality_use_single_dictionary_for_part = 1`. Possible values: diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 2c6e0f05fb5..da1c56e3daf 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1616,6 +1616,63 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; - [Обработка значения NULL в операторе IN](../../sql-reference/operators/in.md#in-null-processing) +## low\_cardinality\_max\_dictionary\_size {#low_cardinality_max_dictionary_size} + +Задает максимальную длину строк в общем глобальном словаре для типа данных `LowCardinality`, который может быть записан в файловую систему хранилища. Настройка предоствращает проблемы с оперативной памятью в случае неограниченного увеличения словаря. Все данные, которые не могут быть закодированы из-за ограничения максимального размера словаря, ClickHouse записывает обычным способом. + +Допустимые значения: + +- Положительное целое число. + +Значение по умолчанию: 8192. + +## low\_cardinality\_use\_single\_dictionary\_for\_part {#low_cardinality_use_single_dictionary_for_part} + +Включает или выключает использование единого словаря для частей данных. + +По умолчанию сервер ClickHouse следит за размером словарей, и если словарь переполняется, сервер создает следующий. Чтобы запретить создание нескольких словарей, задайте настройку `low_cardinality_use_single_dictionary_for_part = 1`. + +Допустимые значения: + +- 1 — Создание нескольких словарей для частей данных запрещено. +- 0 — Создание нескольких словарей для частей данных не запрещено. + +Значение по умолчанию: 0. + +## low\_cardinality\_allow\_in\_native\_format {#low_cardinality_allow_in_native_format} + +Разрешает или запрещает использование типа данных `LowCardinality` с форматом данных [Native](../../interfaces/formats.md#native). + +Если использование типа `LowCardinality` ограничено, сервер CLickHouse преобразует столбцы `LowCardinality` в обычные столбцы для запросов `SELECT`, а обычные столбцы - в столбцы `LowCardinality` для запросов `INSERT`. + +В основном настройка используется для сторонних клиентов, не поддерживающих тип данных `LowCardinality`. + +Допустимые значения: + +- 1 — Использование `LowCardinality` не ограничено. +- 0 — Использование `LowCardinality` ограничено. + +Значение по умолчанию: 1. + +## allow\_suspicious\_low\_cardinality\_types {#allow_suspicious_low_cardinality_types} + +Разрешает или запрещает использование типа данных `LowCardinality` с типами данных с фиксированным размером 8 байт или меньше: числовые типы данных и `FixedString (8_bytes_or_less)`. + +Для небольших фиксированных значений использование `LowCardinality` обычно неэффективно, поскольку ClickHouse хранит числовой индекс для каждой строки. В результате: + +- Используется больше дискового пространства. +- Потребление ОЗУ увеличивается, в зависимости от размера словаря. +- Некоторые функции работают медленнее из-за дополнительных операций кодирования. + +Время слияния в таблицах на движке [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) также может увеличиться по описанным выше причинам. + +Допустимые значения: + +- 1 — Использование `LowCardinality` не ограничено. +- 0 — Использование `LowCardinality` ограничено. + +Значение по умолчанию: 0. + ## background_buffer_flush_schedule_pool_size {#background_buffer_flush_schedule_pool_size} Задает количество потоков для выполнения фонового сброса данных в таблицах с движком [Buffer](../../engines/table-engines/special/buffer.md). Настройка применяется при запуске сервера ClickHouse и не может быть изменена в пользовательском сеансе. diff --git a/docs/ru/sql-reference/data-types/aggregatefunction.md b/docs/ru/sql-reference/data-types/aggregatefunction.md index 07983885bde..22825deb3eb 100644 --- a/docs/ru/sql-reference/data-types/aggregatefunction.md +++ b/docs/ru/sql-reference/data-types/aggregatefunction.md @@ -1,3 +1,8 @@ +--- +toc_priority: 53 +toc_title: AggregateFunction +--- + # AggregateFunction {#data-type-aggregatefunction} Агрегатные функции могут обладать определяемым реализацией промежуточным состоянием, которое может быть сериализовано в тип данных, соответствующий AggregateFunction(…), и быть записано в таблицу обычно посредством [материализованного представления] (../../sql-reference/statements/create.md#create-view). Чтобы получить промежуточное состояние, обычно используются агрегатные функции с суффиксом `-State`. Чтобы в дальнейшем получить агрегированные данные необходимо использовать те же агрегатные функции с суффиксом `-Merge`. diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index 09973d8162c..906246b66ee 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -1,3 +1,8 @@ +--- +toc_priority: 52 +toc_title: Array(T) +--- + # Array(T) {#data-type-array} Массив из элементов типа `T`. diff --git a/docs/ru/sql-reference/data-types/lowcardinality.md b/docs/ru/sql-reference/data-types/lowcardinality.md new file mode 100644 index 00000000000..fc10624600a --- /dev/null +++ b/docs/ru/sql-reference/data-types/lowcardinality.md @@ -0,0 +1,59 @@ +--- +toc_priority: 51 +toc_title: LowCardinality +--- + +# LowCardinality {#lowcardinality-data-type} + +Изменяет внутреннее представление других типов данных, превращая их в тип со словарным кодированием. + +## Синтаксис {#lowcardinality-syntax} + +```sql +LowCardinality(data_type) +``` + +**Параметры** + +- `data_type` — [String](string.md), [FixedString](fixedstring.md), [Date](date.md), [DateTime](datetime.md) и числа за исключением типа [Decimal](decimal.md). `LowCardinality` неэффективен для некоторых типов данных, см. описание настройки [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types). + +## Описание {#lowcardinality-dscr} + +`LowCardinality` — это надстройка, изменяющая способ хранения и правила обработки данных. ClickHouse применяет [словарное кодирование](https://en.wikipedia.org/wiki/Dictionary_coder) в столбцы типа `LowCardinality`. Работа с данными, представленными в словарном виде, значительно увеличивает производительность запросов [SELECT](../statements/select/index.md) для многих приложений. + +Эффективность использования типа данных `LowCarditality` зависит от разнообразия данных. Если словарь содержит менее 10 000 различных значений, ClickHouse в основном показывает более высокую эффективность чтения и хранения данных. Если же словарь содержит более 100 000 различных значений, ClickHouse может работать хуже, чем при использовании обычных типов данных. + +При работе со строками используйте `LowCardinality` вместо [Enum](enum.md). `LowCardinality` обеспечивает большую гибкость в использовании и часто показывает такую же или более высокую эффективность. + +## Пример + +Создать таблицу со столбцами типа `LowCardinality`: + +```sql +CREATE TABLE lc_t +( + `id` UInt16, + `strings` LowCardinality(String) +) +ENGINE = MergeTree() +ORDER BY id +``` + +## Связанные настройки и функции + +Настройки: + +- [low_cardinality_max_dictionary_size](../../operations/settings/settings.md#low_cardinality_max_dictionary_size) +- [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part) +- [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format) +- [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) + +Функции: + +- [toLowCardinality](../functions/type-conversion-functions.md#tolowcardinality) + +## Смотрите также + +- [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). +- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). +- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). \ No newline at end of file diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index 5ed99469750..71e1f7a37a0 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -1,3 +1,8 @@ +--- +toc_priority: 55 +toc_title: Nullable +--- + # Nullable(TypeName) {#data_type-nullable} Позволяет работать как со значением типа `TypeName` так и с отсутствием этого значения ([NULL](../../sql-reference/data-types/nullable.md)) в одной и той же переменной, в том числе хранить `NULL` в таблицах вместе со значения типа `TypeName`. Например, в столбце типа `Nullable(Int8)` можно хранить значения типа `Int8`, а в тех строках, где значения нет, будет храниться `NULL`. diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index 566a582eb95..cb8130f28a3 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -1,3 +1,8 @@ +--- +toc_priority: 54 +toc_title: Tuple(T1, T2, ...) +--- + # Tuple(T1, T2, …) {#tuplet1-t2} Кортеж из элементов любого [типа](index.md#data_types). Элементы кортежа могут быть одного или разных типов. diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 41ded78055c..3b70f0d6577 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -508,9 +508,48 @@ SELECT parseDateTimeBestEffort('10 20:19') **См. также** -- \[Информация о формате ISO 8601 от @xkcd\](https://xkcd.com/1179/) +- [Информация о формате ISO 8601 от @xkcd](https://xkcd.com/1179/) - [RFC 1123](https://tools.ietf.org/html/rfc1123) - [toDate](#todate) - [toDateTime](#todatetime) +## toLowCardinality {#tolowcardinality} + +Преобразует входные данные в версию [LowCardianlity](../data-types/lowcardinality.md) того же типа данных. + +Чтобы преобразовать данные из типа `LowCardinality`, используйте функцию [CAST](#type_conversion_function-cast). Например, `CAST(x as String)`. + +**Синтаксис** + +```sql +toLowCardinality(expr) +``` + +**Параметры** + +- `expr` — [Выражение](../syntax.md#syntax-expressions), которое в результате преобразуется в один из [поддерживаемых типов данных](../data-types/index.md#data_types). + + +**Возвращаемое значение** + +- Результат преобразования `expr`. + +Тип: `LowCardinality(expr_result_type)` + +**Example** + +Запрос: + +```sql +SELECT toLowCardinality('1') +``` + +Результат: + +```text +┌─toLowCardinality('1')─┐ +│ 1 │ +└───────────────────────┘ +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/type_conversion_functions/) From 6ba9c107df75d21c9a4642c67b8552c9bcb7bcf2 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sat, 12 Sep 2020 20:34:08 +0300 Subject: [PATCH 061/390] add EN changes --- docs/en/sql-reference/data-types/lowcardinality.md | 2 +- docs/en/sql-reference/functions/type-conversion-functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md index 7ccac61e4d7..1a0cedb99c7 100644 --- a/docs/en/sql-reference/data-types/lowcardinality.md +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -21,7 +21,7 @@ LowCardinality(data_type) `LowCardinality` is a superstructure that changes a data storage method and rules of data processing. ClickHouse applies [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) to `LowCardinality`-columns. Operating with dictionary encoded data significantly increases performance of [SELECT](../../sql-reference/statements/select/index.md) queries for many applications. -The efficiency of using `LowCarditality` data type depends on data diversity. If a dictionary contains less than 10,000 distinct values, then ClickHouse mostly shows higher efficiency of data reading and storing. If a dictionary contains more than 100,000 distinct values, then ClickHouse can perform worse in comparison with using ordinary data types. +The efficiency of using `LowCardinality` data type depends on data diversity. If a dictionary contains less than 10,000 distinct values, then ClickHouse mostly shows higher efficiency of data reading and storing. If a dictionary contains more than 100,000 distinct values, then ClickHouse can perform worse in comparison with using ordinary data types. Consider using `LowCardinality` instead of [Enum](../../sql-reference/data-types/enum.md) when working with strings. `LowCardinality` provides more flexibility in use and often reveals the same or higher efficiency. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 67361c350c7..e466c025d80 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -516,7 +516,7 @@ Result: **See Also** -- \[ISO 8601 announcement by @xkcd\](https://xkcd.com/1179/) +- [ISO 8601 announcement by @xkcd](https://xkcd.com/1179/) - [RFC 1123](https://tools.ietf.org/html/rfc1123) - [toDate](#todate) - [toDateTime](#todatetime) From 882b2a33488c4dbabbe96fff40c01a065fe0a860 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 13 Sep 2020 01:00:04 +0800 Subject: [PATCH 062/390] CTE --- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 90 +++++++++++++++++++ src/Interpreters/ApplyWithSubqueryVisitor.h | 30 +++++++ .../ExecuteScalarSubqueriesVisitor.cpp | 5 ++ src/Interpreters/InterpreterSelectQuery.cpp | 3 + src/Interpreters/ya.make | 1 + src/Parsers/ASTWithElement.cpp | 21 +++++ src/Parsers/ASTWithElement.h | 25 ++++++ src/Parsers/ParserSelectQuery.cpp | 6 +- src/Parsers/ParserWithElement.cpp | 39 ++++++++ src/Parsers/ParserWithElement.h | 18 ++++ src/Parsers/ya.make | 2 + ...495_subqueries_in_with_statement.reference | 14 +++ .../01495_subqueries_in_with_statement.sql | 13 +++ 13 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 src/Interpreters/ApplyWithSubqueryVisitor.cpp create mode 100644 src/Interpreters/ApplyWithSubqueryVisitor.h create mode 100644 src/Parsers/ASTWithElement.cpp create mode 100644 src/Parsers/ASTWithElement.h create mode 100644 src/Parsers/ParserWithElement.cpp create mode 100644 src/Parsers/ParserWithElement.h create mode 100644 tests/queries/0_stateless/01495_subqueries_in_with_statement.reference create mode 100644 tests/queries/0_stateless/01495_subqueries_in_with_statement.sql diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp new file mode 100644 index 00000000000..e03682dafb3 --- /dev/null +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) +{ + if (auto * node_select = ast->as()) + { + auto with = node_select->with(); + std::optional new_data; + if (with) + { + for (auto & child : with->children) + visit(child, data); + for (auto & child : with->children) + { + if (auto * ast_with_elem = child->as()) + { + if (!new_data) + new_data = data; + new_data->subqueries[ast_with_elem->name] = ast_with_elem->subquery; + } + } + } + + for (auto & child : node_select->children) + { + if (child != with) + visit(child, new_data ? *new_data : data); + } + return; + } + + for (auto & child : ast->children) + visit(child, data); + if (auto * node_func = ast->as()) + visit(*node_func, data); + else if (auto * node_table = ast->as()) + visit(*node_table, data); +} + +void ApplyWithSubqueryVisitor::visit(ASTTableExpression & table, const Data & data) +{ + if (table.database_and_table_name) + { + auto table_id = IdentifierSemantic::extractDatabaseAndTable(table.database_and_table_name->as()); + if (table_id.database_name.empty()) + { + auto subquery_it = data.subqueries.find(table_id.table_name); + if (subquery_it != data.subqueries.end()) + { + table.children.clear(); + table.database_and_table_name.reset(); + table.subquery = subquery_it->second->clone(); + dynamic_cast(*table.subquery).alias = table_id.table_name; + table.children.emplace_back(table.subquery); + } + } + } +} + +void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data) +{ + if (checkFunctionIsInOrGlobalInOperator(func)) + { + auto & ast = func.arguments->children.at(1); + if (const auto * ident = ast->as()) + { + auto table_id = IdentifierSemantic::extractDatabaseAndTable(*ident); + if (table_id.database_name.empty()) + { + auto subquery_it = data.subqueries.find(table_id.table_name); + if (subquery_it != data.subqueries.end()) + { + func.arguments->children[1] = subquery_it->second->clone(); + dynamic_cast(*func.arguments->children[1]).alias = table_id.table_name; + } + } + } + } +} + +} diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.h b/src/Interpreters/ApplyWithSubqueryVisitor.h new file mode 100644 index 00000000000..2aecd6aee01 --- /dev/null +++ b/src/Interpreters/ApplyWithSubqueryVisitor.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include + +namespace DB +{ +// TODO After we support `union_with_global`, this visitor should also be extended to match ASTSelectQueryWithUnion. +class ASTSelectQuery; +class ASTFunction; +struct ASTTableExpression; + +class ApplyWithSubqueryVisitor +{ +public: + struct Data + { + std::map subqueries; + }; + + static void visit(ASTPtr & ast) { visit(ast, {}); } + +private: + static void visit(ASTPtr & ast, const Data & data); + static void visit(ASTTableExpression & table, const Data & data); + static void visit(ASTFunction & func, const Data & data); +}; + +} diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index ee29d301c6b..f7a1fc83182 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -40,6 +41,10 @@ bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr if (node->as()) return false; + /// Do not go to subqueries defined in with statement + if (node->as()) + return false; + if (node->as()) { /// Do not go to FROM, JOIN, UNION. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index dbf6d5ae8d3..603476ac1ba 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -249,6 +250,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( source_header = input_pipe->getHeader(); } + ApplyWithSubqueryVisitor().visit(query_ptr); + JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery()); if (!has_input && !storage) diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 4eacdab1dcf..5f520505a8a 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -23,6 +23,7 @@ SRCS( addTypeConversionToAST.cpp AggregateDescription.cpp Aggregator.cpp + ApplyWithSubqueryVisitor.cpp ArithmeticOperationsInAgrFuncOptimize.cpp ArrayJoinAction.cpp AsynchronousMetricLog.cpp diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp new file mode 100644 index 00000000000..e8dd4ff0498 --- /dev/null +++ b/src/Parsers/ASTWithElement.cpp @@ -0,0 +1,21 @@ +#include + +namespace DB +{ + +ASTPtr ASTWithElement::clone() const +{ + const auto res = std::make_shared(*this); + res->name = name; + res->subquery = subquery->clone(); + res->children.emplace_back(res->subquery); + return res; +} + +void ASTWithElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + settings.writeIdentifier(name); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); + subquery->formatImpl(settings, state, frame); +} +} diff --git a/src/Parsers/ASTWithElement.h b/src/Parsers/ASTWithElement.h new file mode 100644 index 00000000000..97c68579fa1 --- /dev/null +++ b/src/Parsers/ASTWithElement.h @@ -0,0 +1,25 @@ +#pragma once + +#include + + +namespace DB +{ +/** subquery in with statement + */ +class ASTWithElement : public IAST +{ +public: + String name; + ASTPtr subquery; + + /** Get the text that identifies this element. */ + String getID(char) const override { return "WithElement"; } + + ASTPtr clone() const override; + +protected: + void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index d2d7bbf9f21..9f2df82b4b4 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -74,7 +75,10 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (s_with.ignore(pos, expected)) { - if (!exp_list_for_with_clause.parse(pos, with_expression_list, expected)) + if (!ParserList(std::make_unique(), std::make_unique(TokenType::Comma)) + .parse(pos, with_expression_list, expected)) + return false; + if (with_expression_list->children.empty()) return false; } } diff --git a/src/Parsers/ParserWithElement.cpp b/src/Parsers/ParserWithElement.cpp new file mode 100644 index 00000000000..048e891f0df --- /dev/null +++ b/src/Parsers/ParserWithElement.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +bool ParserWithElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserIdentifier s_ident; + ParserKeyword s_as("AS"); + ParserSubquery s_subquery; + + auto old_pos = pos; + if (ASTPtr name, subquery; + s_ident.parse(pos, name, expected) && s_as.ignore(pos, expected) && s_subquery.parse(pos, subquery, expected)) + { + auto with_element = std::make_shared(); + tryGetIdentifierNameInto(name, with_element->name); + with_element->subquery = subquery; + node = with_element; + } + else + { + pos = old_pos; + ParserExpressionWithOptionalAlias s_expr(false); + if (!s_expr.parse(pos, node, expected)) + return false; + } + return true; +} + + +} diff --git a/src/Parsers/ParserWithElement.h b/src/Parsers/ParserWithElement.h new file mode 100644 index 00000000000..75ad11f5deb --- /dev/null +++ b/src/Parsers/ParserWithElement.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +/** WITH (scalar query) AS identifier + * or WITH identifier AS (subquery) + */ +class ParserWithElement : public IParserBase +{ +protected: + const char * getName() const override { return "WITH element"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index fabf2bbb8fd..0a0c301b722 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -61,6 +61,7 @@ SRCS( ASTTTLElement.cpp ASTUserNameWithHost.cpp ASTWithAlias.cpp + ASTWithElement.cpp CommonParsers.cpp ExpressionElementParsers.cpp ExpressionListParsers.cpp @@ -133,6 +134,7 @@ SRCS( ParserUseQuery.cpp ParserUserNameWithHost.cpp ParserWatchQuery.cpp + ParserWithElement.cpp parseUserName.cpp queryToString.cpp QueryWithOutputSettingsPushDownVisitor.cpp diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference b/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference new file mode 100644 index 00000000000..8e851cd3ba1 --- /dev/null +++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference @@ -0,0 +1,14 @@ +0 +1 +2 +3 +4 +2 3 +4 5 +2 3 +4 5 +1 1 2 +3 3 4 +4 5 +4 5 +4 5 diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql new file mode 100644 index 00000000000..9ec921a9d4c --- /dev/null +++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS test1; + +CREATE TABLE test1(i int, j int) ENGINE Log; + +INSERT INTO test1 VALUES (1, 2), (3, 4); + +WITH test1 AS (SELECT * FROM numbers(5)) SELECT * FROM test1; +WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT * FROM test1; +WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT * FROM (SELECT * FROM test1); +SELECT * FROM (WITH test1 AS (SELECT toInt32(*) i FROM numbers(5)) SELECT * FROM test1) l ANY INNER JOIN test1 r on (l.i == r.i); +WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT toInt64(4) i, toInt64(5) j FROM numbers(3) WHERE (i, j) IN test1; + +DROP TABLE IF EXISTS test1; From ac9ba23bdfa67bd0188ec00ccbff9816bc981bd5 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 14 Sep 2020 15:49:04 +0300 Subject: [PATCH 063/390] fix more tests --- tests/integration/test_check_table/test.py | 3 ++- .../configs/wide_parts_only.xml | 6 ++++++ .../test_default_compression_codec/test.py | 6 +++--- .../configs/config.d/storage_conf.xml | 4 ++++ .../test_merge_tree_s3_with_cache/test.py | 21 ++++++++++--------- .../configs/wide_parts_only.xml | 6 ++++++ .../test_mutations_hardlinks/test.py | 2 +- 7 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_default_compression_codec/configs/wide_parts_only.xml create mode 100644 tests/integration/test_mutations_hardlinks/configs/wide_parts_only.xml diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py index 83df59b44a0..f972e7a92ba 100644 --- a/tests/integration/test_check_table/test.py +++ b/tests/integration/test_check_table/test.py @@ -24,7 +24,8 @@ def started_cluster(): node1.query(''' CREATE TABLE non_replicated_mt(date Date, id UInt32, value Int32) - ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY id; + ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY id + SETTINGS min_bytes_for_wide_part=0; ''') yield cluster diff --git a/tests/integration/test_default_compression_codec/configs/wide_parts_only.xml b/tests/integration/test_default_compression_codec/configs/wide_parts_only.xml new file mode 100644 index 00000000000..42e2173f718 --- /dev/null +++ b/tests/integration/test_default_compression_codec/configs/wide_parts_only.xml @@ -0,0 +1,6 @@ + + + 0 + 0 + + diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index d312a93ba01..0cfbb0b67cf 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -6,9 +6,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True) @pytest.fixture(scope="module") def start_cluster(): diff --git a/tests/integration/test_merge_tree_s3_with_cache/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_with_cache/configs/config.d/storage_conf.xml index b32770095fc..f3b7f959ce9 100644 --- a/tests/integration/test_merge_tree_s3_with_cache/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_with_cache/configs/config.d/storage_conf.xml @@ -18,4 +18,8 @@ + + + 0 + diff --git a/tests/integration/test_merge_tree_s3_with_cache/test.py b/tests/integration/test_merge_tree_s3_with_cache/test.py index 25c08777ae5..d5d6db2fb77 100644 --- a/tests/integration/test_merge_tree_s3_with_cache/test.py +++ b/tests/integration/test_merge_tree_s3_with_cache/test.py @@ -40,7 +40,8 @@ def get_query_stat(instance, hint): return result -def test_write_is_cached(cluster): +@pytest.mark.parametrize("min_rows_for_wide_part,read_requests", [(0, 2), (8192, 1)]) +def test_write_is_cached(cluster, min_rows_for_wide_part, read_requests): node = cluster.instances["node"] node.query( @@ -50,8 +51,8 @@ def test_write_is_cached(cluster): data String ) ENGINE=MergeTree() ORDER BY id - SETTINGS storage_policy='s3' - """ + SETTINGS storage_policy='s3', min_rows_for_wide_part={} + """.format(min_rows_for_wide_part) ) node.query("SYSTEM FLUSH LOGS") @@ -63,12 +64,12 @@ def test_write_is_cached(cluster): assert node.query(select_query) == "(0,'data'),(1,'data')" stat = get_query_stat(node, select_query) - assert stat["S3ReadRequestsCount"] == 2 # Only .bin files should be accessed from S3. + assert stat["S3ReadRequestsCount"] == read_requests # Only .bin files should be accessed from S3. node.query("DROP TABLE IF EXISTS s3_test NO DELAY") - -def test_read_after_cache_is_wiped(cluster): +@pytest.mark.parametrize("min_rows_for_wide_part,all_files,bin_files", [(0, 4, 2), (8192, 2, 1)]) +def test_read_after_cache_is_wiped(cluster, min_rows_for_wide_part, all_files, bin_files): node = cluster.instances["node"] node.query( @@ -78,8 +79,8 @@ def test_read_after_cache_is_wiped(cluster): data String ) ENGINE=MergeTree() ORDER BY id - SETTINGS storage_policy='s3' - """ + SETTINGS storage_policy='s3', min_rows_for_wide_part={} + """.format(min_rows_for_wide_part) ) node.query("SYSTEM FLUSH LOGS") @@ -93,12 +94,12 @@ def test_read_after_cache_is_wiped(cluster): select_query = "SELECT * FROM s3_test" node.query(select_query) stat = get_query_stat(node, select_query) - assert stat["S3ReadRequestsCount"] == 4 # .mrk and .bin files should be accessed from S3. + assert stat["S3ReadRequestsCount"] == all_files # .mrk and .bin files should be accessed from S3. # After cache is populated again, only .bin files should be accessed from S3. select_query = "SELECT * FROM s3_test order by id FORMAT Values" assert node.query(select_query) == "(0,'data'),(1,'data')" stat = get_query_stat(node, select_query) - assert stat["S3ReadRequestsCount"] == 2 + assert stat["S3ReadRequestsCount"] == bin_files node.query("DROP TABLE IF EXISTS s3_test NO DELAY") diff --git a/tests/integration/test_mutations_hardlinks/configs/wide_parts_only.xml b/tests/integration/test_mutations_hardlinks/configs/wide_parts_only.xml new file mode 100644 index 00000000000..42e2173f718 --- /dev/null +++ b/tests/integration/test_mutations_hardlinks/configs/wide_parts_only.xml @@ -0,0 +1,6 @@ + + + 0 + 0 + + diff --git a/tests/integration/test_mutations_hardlinks/test.py b/tests/integration/test_mutations_hardlinks/test.py index 56852f572ff..4e70e76bc63 100644 --- a/tests/integration/test_mutations_hardlinks/test.py +++ b/tests/integration/test_mutations_hardlinks/test.py @@ -9,7 +9,7 @@ from multiprocessing.dummy import Pool cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') +node1 = cluster.add_instance('node1', main_configs=['configs/wide_parts_only.xml']) @pytest.fixture(scope="module") def started_cluster(): From 5697f6d926c2dc04892aca3ef7b8297ef91d8da6 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 14 Sep 2020 16:14:18 +0300 Subject: [PATCH 064/390] style fix --- src/Columns/ColumnVector.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index c548ce3ca5c..4d7b7856363 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -19,7 +19,6 @@ #include - #if !defined(ARCADIA_BUILD) # include # if USE_OPENCL @@ -219,7 +218,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi if (isNaN(data[res[reverse ? i : s - 1 - i]])) ++nans_to_move; else - break; + break;completeThread } if (nans_to_move) From 2bffefae1ac4ceb5fe48a5b445a6b1ac4dfe6ff7 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 14 Sep 2020 16:30:44 +0300 Subject: [PATCH 065/390] typo --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 4d7b7856363..a09f64ad580 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -218,7 +218,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi if (isNaN(data[res[reverse ? i : s - 1 - i]])) ++nans_to_move; else - break;completeThread + break; } if (nans_to_move) From 3795dfed144c93e57486c7d0ab5d370e9e8cc82b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Sep 2020 16:33:36 +0300 Subject: [PATCH 066/390] Remove one header --- base/common/CMakeLists.txt | 1 + base/common/StringRef.cpp | 13 +++++++++++++ base/common/StringRef.h | 10 ++-------- base/common/ya.make | 1 + src/Columns/ya.make | 2 ++ src/Common/ya.make | 1 + 6 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 base/common/StringRef.cpp diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index 903168a0dd4..9e4462c821a 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -18,6 +18,7 @@ set (SRCS terminalColors.cpp errnoToString.cpp getResource.cpp + StringRef.cpp ) if (ENABLE_REPLXX) diff --git a/base/common/StringRef.cpp b/base/common/StringRef.cpp new file mode 100644 index 00000000000..87877360d83 --- /dev/null +++ b/base/common/StringRef.cpp @@ -0,0 +1,13 @@ +#include + +#include "StringRef.h" + + +std::ostream & operator<<(std::ostream & os, const StringRef & str) +{ + if (str.data) + os.write(str.data, str.size); + + return os; +} + diff --git a/base/common/StringRef.h b/base/common/StringRef.h index 410e13ba7d8..05d4eda7656 100644 --- a/base/common/StringRef.h +++ b/base/common/StringRef.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -322,10 +322,4 @@ inline bool operator==(StringRef lhs, const char * rhs) return true; } -inline std::ostream & operator<<(std::ostream & os, const StringRef & str) -{ - if (str.data) - os.write(str.data, str.size); - - return os; -} +std::ostream & operator<<(std::ostream & os, const StringRef & str); diff --git a/base/common/ya.make b/base/common/ya.make index 2bd08afbf3a..cbb6b5f64ac 100644 --- a/base/common/ya.make +++ b/base/common/ya.make @@ -53,6 +53,7 @@ SRCS( setTerminalEcho.cpp shift10.cpp sleep.cpp + StringRef.cpp terminalColors.cpp ) diff --git a/src/Columns/ya.make b/src/Columns/ya.make index 910c479c2a9..78c0e1b992d 100644 --- a/src/Columns/ya.make +++ b/src/Columns/ya.make @@ -2,6 +2,8 @@ LIBRARY() ADDINCL( + contrib/libs/icu/common + contrib/libs/icu/i18n contrib/libs/pdqsort ) diff --git a/src/Common/ya.make b/src/Common/ya.make index d9a7a2ce4de..2478fa0c9ce 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -86,6 +86,7 @@ SRCS( StatusFile.cpp StatusInfo.cpp Stopwatch.cpp + StringRef.cpp StringUtils/StringUtils.cpp StudentTTest.cpp SymbolIndex.cpp From f725f8deee7fb8d695e1e3282a8b830a95ccf6ed Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 15 Sep 2020 02:14:14 +0300 Subject: [PATCH 067/390] fix more tests --- .../test_backward_compatibility/configs/wide_parts_only.xml | 5 +++++ tests/integration/test_backward_compatibility/test.py | 4 ++-- .../0_stateless/00804_test_alter_compression_codecs.sql | 2 +- .../0_stateless/00926_adaptive_index_granularity_pk.sql | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_backward_compatibility/configs/wide_parts_only.xml diff --git a/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml new file mode 100644 index 00000000000..b240c0fcb2a --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/wide_parts_only.xml @@ -0,0 +1,5 @@ + + + 0 + + diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 5b51823d361..cef70add3d0 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -5,7 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/wide_parts_only.xml'], with_zookeeper=True) @pytest.fixture(scope="module") def start_cluster(): @@ -24,7 +24,7 @@ def start_cluster(): cluster.shutdown() -def test_backward_compatability(start_cluster): +def test_backward_compatability1(start_cluster): node2.query("INSERT INTO t VALUES (today(), 1)") node1.query("SYSTEM SYNC REPLICA t", timeout=10) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index a9e6c12735c..4710694baf5 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -63,7 +63,7 @@ CREATE TABLE large_alter_table_00804 ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2 SETTINGS min_bytes_for_wide_part = 0; INSERT INTO large_alter_table_00804 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql index b71c0640bd5..fe434845c29 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql @@ -62,7 +62,7 @@ CREATE TABLE large_alter_table_00926 ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity_bytes=40, min_index_granularity_bytes=30, write_final_mark = 0; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS min_index_granularity_bytes=30, write_final_mark = 0, min_bytes_for_wide_part = '10M'; INSERT INTO large_alter_table_00926 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; From 52f921a6f98772ba75d34df77195a840fec37758 Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Tue, 15 Sep 2020 12:24:03 +0300 Subject: [PATCH 068/390] CLICKHOUSEDOCS-758: Fixed links --- docs/ru/interfaces/formats.md | 4 ++-- docs/ru/operations/settings/settings.md | 2 +- .../aggregate-functions/reference/groupbitmap.md | 2 +- docs/ru/sql-reference/functions/bitmap-functions.md | 8 ++++---- docs/ru/sql-reference/functions/random-functions.md | 1 + docs/tools/test.py | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 04bca115974..dd68f7eb646 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1050,13 +1050,13 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../engines/table-engines/integrations/hdfs.md). -## Arrow {data-format-arrow} +## Arrow {#data-format-arrow} [Apache Arrow](https://arrow.apache.org/) поставляется с двумя встроенными поколоночнами форматами хранения. ClickHouse поддерживает операции чтения и записи для этих форматов. `Arrow` — это Apache Arrow's "file mode" формат. Он предназначен для произвольного доступа в памяти. -## ArrowStream {data-format-arrow-stream} +## ArrowStream {#data-format-arrow-stream} `ArrowStream` — это Apache Arrow's "stream mode" формат. Он предназначен для обработки потоков в памяти. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 2c6e0f05fb5..333c827fe97 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -484,7 +484,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( См. также: -- [JOIN strictness](../../sql-reference/statements/select/join.md#select-join-strictness) +- [JOIN strictness](../../sql-reference/statements/select/join.md#join-settings) ## max\_block\_size {#setting-max_block_size} diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md index c01636e155d..a4be18b75ec 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md @@ -4,7 +4,7 @@ toc_priority: 128 # groupBitmap {#groupbitmap} -Bitmap или агрегатные вычисления для столбца с типом данных `UInt*`, возвращают кардинальность в виде значения типа UInt64, если добавить суффикс -State, то возвращают [объект bitmap](../../../sql-reference/functions/bitmap-functions.md). +Bitmap или агрегатные вычисления для столбца с типом данных `UInt*`, возвращают кардинальность в виде значения типа UInt64, если добавить суффикс `-State`, то возвращают [объект bitmap](../../../sql-reference/functions/bitmap-functions.md#bitmap-functions). ``` sql groupBitmap(expr) diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index c91725c7a39..c5b0646aa79 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -1,4 +1,4 @@ -# Функции для битмапов {#funktsii-dlia-bitmapov} +# Функции для битмапов {#bitmap-functions} ## bitmapBuild {#bitmap_functions-bitmapbuild} @@ -61,8 +61,8 @@ bitmapSubsetLimit(bitmap, range_start, cardinality_limit) **Параметры** - `bitmap` – Битмап. [Bitmap object](#bitmap_functions-bitmapbuild). -- `range_start` – Начальная точка подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md). -- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md). +- `range_start` – Начальная точка подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). +- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). **Возвращаемое значение** @@ -97,7 +97,7 @@ bitmapContains(haystack, needle) **Параметры** - `haystack` – [объект Bitmap](#bitmap_functions-bitmapbuild), в котором функция ищет значение. -- `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql-reference/functions/bitmap-functions.md). +- `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). **Возвращаемые значения** diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index 4aaaef5cb5d..21dcfeeb3c0 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -100,5 +100,6 @@ FROM numbers(3) │ a*cjab+ │ │ aeca2A │ └───────────────────────────────────────┘ +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/random_functions/) diff --git a/docs/tools/test.py b/docs/tools/test.py index 5c0cf4b799d..d963d34df08 100755 --- a/docs/tools/test.py +++ b/docs/tools/test.py @@ -92,7 +92,7 @@ def test_single_page(input_path, lang): logging.warning('Found %d duplicate anchor points' % duplicate_anchor_points) if links_to_nowhere: - if lang == 'en': # TODO: check all languages again + if lang == 'en' or lang == 'ru': # TODO: check all languages again logging.error(f'Found {links_to_nowhere} links to nowhere in {lang}') sys.exit(1) else: From d943bac1a482276812ac03b6ba161dc7f4bab648 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 15 Sep 2020 13:29:47 +0300 Subject: [PATCH 069/390] Exception on double init of global thread pool --- src/Common/ThreadPool.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 49516d777fb..93aa6be8d9a 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -13,6 +13,7 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_SCHEDULE_TASK; + extern const int LOGICAL_ERROR; } } @@ -276,7 +277,11 @@ std::unique_ptr GlobalThreadPool::the_instance; void GlobalThreadPool::initialize(size_t max_threads) { - assert(!the_instance); + if (the_instance) + { + throw Exception(LOGICAL_ERROR, + "The global thread pool is initialized twice"); + } the_instance.reset(new GlobalThreadPool(max_threads, 1000 /*max_free_threads*/, 10000 /*max_queue_size*/, From 63db2ca68d9797d5c6f5242350d55fdff6c88f05 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 15 Sep 2020 15:30:07 +0300 Subject: [PATCH 070/390] fix test --- .../queries/0_stateless/00804_test_alter_compression_codecs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index 4710694baf5..2a1b9e55b9a 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -63,7 +63,7 @@ CREATE TABLE large_alter_table_00804 ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2 SETTINGS min_bytes_for_wide_part = 0; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; INSERT INTO large_alter_table_00804 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; From 4c783f19ee388fd7e5622e1585f91adb004cff29 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Sep 2020 16:25:14 +0300 Subject: [PATCH 071/390] Use QueryPlan for SubqueryForSet. --- src/Interpreters/ActionsVisitor.cpp | 5 +- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/GlobalSubqueriesVisitor.h | 3 +- src/Interpreters/InterpreterSelectQuery.cpp | 38 +++++++-- src/Interpreters/SubqueryForSet.cpp | 5 +- src/Interpreters/SubqueryForSet.h | 4 +- src/Processors/QueryPipeline.cpp | 16 +++- src/Processors/QueryPipeline.h | 2 + src/Processors/QueryPlan/CreatingSetsStep.cpp | 84 +++++++++++++++---- src/Processors/QueryPlan/CreatingSetsStep.h | 26 +++++- 10 files changed, 150 insertions(+), 35 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 9d6d5f783ff..be040ff2c34 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -900,10 +900,11 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su * in the subquery_for_set object, this subquery is set as source and the temporary table _data1 as the table. * - this function shows the expression IN_data1. */ - if (subquery_for_set.source.empty() && data.no_storage_or_local) + if (!subquery_for_set.source && data.no_storage_or_local) { auto interpreter = interpretSubquery(right_in_operand, data.context, data.subquery_depth, {}); - subquery_for_set.source = QueryPipeline::getPipe(interpreter->execute().pipeline); + subquery_for_set.source = std::make_unique(); + interpreter->buildQueryPlan(*subquery_for_set.source); } subquery_for_set.set = set; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index d9fc44d9b8f..14a50c2cfc6 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -582,7 +582,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin(const ASTTablesInSelectQuer ExpressionActionsPtr joined_block_actions = createJoinedBlockActions(context, analyzedJoin()); Names original_right_columns; - if (subquery_for_join.source.empty()) + if (!subquery_for_join.source) { NamesWithAliases required_columns_with_aliases = analyzedJoin().getRequiredColumns( joined_block_actions->getSampleBlock(), joined_block_actions->getRequiredColumns()); diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index e155a132241..719794f0607 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -135,7 +135,8 @@ public: ast = database_and_table_name; external_tables[external_table_name] = external_storage_holder; - subqueries_for_sets[external_table_name].source = QueryPipeline::getPipe(interpreter->execute().pipeline); + subqueries_for_sets[external_table_name].source = std::make_unique(); + interpreter->buildQueryPlan(*subqueries_for_sets[external_table_name].source); subqueries_for_sets[external_table_name].table = external_storage; /** NOTE If it was written IN tmp_table - the existing temporary (but not external) table, diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 079fc792447..6f1be43b3a1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1862,14 +1862,38 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_p const Settings & settings = context->getSettingsRef(); - auto creating_sets = std::make_unique( - query_plan.getCurrentDataStream(), - std::move(subqueries_for_sets), - SizeLimits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode), - *context); + if (subqueries_for_sets.empty()) + return; - creating_sets->setStepDescription("Create sets for subqueries and joins"); - query_plan.addStep(std::move(creating_sets)); + SizeLimits limits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); + + std::vector plans; + DataStreams input_streams; + input_streams.emplace_back(query_plan.getCurrentDataStream()); + + for (auto & [description, set] : subqueries_for_sets) + { + auto plan = std::move(set.source); + std::string type = (set.join != nullptr) ? "JOIN" + : "subquery"; + + auto creating_set = std::make_unique( + plan->getCurrentDataStream(), + query_plan.getCurrentDataStream().header, + std::move(description), + std::move(set), + limits, + *context); + creating_set->setStepDescription("Create set for " + type); + plan->addStep(std::move(creating_set)); + + input_streams.emplace_back(plan->getCurrentDataStream()); + plans.emplace_back(std::move(*plan)); + } + + auto creating_sets = std::make_unique(std::move(input_streams)); + creating_sets->setStepDescription("Create sets before main query execution"); + query_plan.unitePlans(std::move(creating_sets), std::move(plans)); } diff --git a/src/Interpreters/SubqueryForSet.cpp b/src/Interpreters/SubqueryForSet.cpp index 038ecbbb0b6..e944b76e71c 100644 --- a/src/Interpreters/SubqueryForSet.cpp +++ b/src/Interpreters/SubqueryForSet.cpp @@ -12,9 +12,10 @@ void SubqueryForSet::makeSource(std::shared_ptr NamesWithAliases && joined_block_aliases_) { joined_block_aliases = std::move(joined_block_aliases_); - source = QueryPipeline::getPipe(interpreter->execute().pipeline); + source = std::make_unique(); + interpreter->buildQueryPlan(*source); - sample_block = source.getHeader(); + sample_block = interpreter->getSampleBlock(); renameColumns(sample_block); } diff --git a/src/Interpreters/SubqueryForSet.h b/src/Interpreters/SubqueryForSet.h index d268758c3e8..b44f0f6cf8b 100644 --- a/src/Interpreters/SubqueryForSet.h +++ b/src/Interpreters/SubqueryForSet.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB @@ -14,12 +13,13 @@ namespace DB class InterpreterSelectWithUnionQuery; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +class QueryPlan; /// Information on what to do when executing a subquery in the [GLOBAL] IN/JOIN section. struct SubqueryForSet { /// The source is obtained using the InterpreterSelectQuery subquery. - Pipe source; + std::unique_ptr source; /// If set, build it from result. SetPtr set; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 0b654d0f325..c7ffe8b0c9b 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -204,7 +204,7 @@ void QueryPipeline::addCreatingSetsTransform(SubqueriesForSets subqueries_for_se for (auto & subquery : subqueries_for_sets) { - if (!subquery.second.source.empty()) + if (subquery.second.source) { auto & source = sources.emplace_back(std::move(subquery.second.source)); if (source.numOutputPorts() > 1) @@ -315,6 +315,20 @@ QueryPipeline QueryPipeline::unitePipelines( return pipeline; } +void QueryPipeline::addDelayedPipeline(QueryPipeline pipeline) +{ + pipeline.resize(1); + + auto * collected_processors = pipe.collected_processors; + + Pipes pipes; + pipes.emplace_back(QueryPipeline::getPipe(std::move(pipeline))); + pipes.emplace_back(std::move(pipe)); + pipe = Pipe::unitePipes(std::move(pipes), collected_processors); + + pipe.addTransform(std::make_shared(getHeader(), 2)); +} + void QueryPipeline::setProgressCallback(const ProgressCallback & callback) { for (auto & processor : pipe.processors) diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 45b410ab323..06c67c897be 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -87,6 +87,8 @@ public: size_t max_threads_limit = 0, Processors * collected_processors = nullptr); + void addDelayedPipeline(QueryPipeline); + PipelineExecutorPtr execute(); size_t getNumStreams() const { return pipe.numOutputPorts(); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 7e840e1531b..9a4c11f9222 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -22,37 +22,91 @@ static ITransformingStep::Traits getTraits() }; } -CreatingSetsStep::CreatingSetsStep( +CreatingSetStep::CreatingSetStep( const DataStream & input_stream_, - SubqueriesForSets subqueries_for_sets_, + Block header, + String description_, + SubqueryForSet subquery_for_set_, SizeLimits network_transfer_limits_, const Context & context_) - : ITransformingStep(input_stream_, input_stream_.header, getTraits()) - , subqueries_for_sets(std::move(subqueries_for_sets_)) + : ITransformingStep(input_stream_, header, getTraits()) + , description(std::move(description_)) + , subquery_for_set(std::move(subquery_for_set_)) , network_transfer_limits(std::move(network_transfer_limits_)) , context(context_) { } -void CreatingSetsStep::transformPipeline(QueryPipeline & pipeline) +void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) { - pipeline.addCreatingSetsTransform(std::move(subqueries_for_sets), network_transfer_limits, context); + pipeline.resize(1); + pipeline.addTransform( + std::make_shared( + pipeline.getHeader(), + getOutputStream().header, + std::move(subquery_for_set), + network_transfer_limits, + context)); } -void CreatingSetsStep::describeActions(FormatSettings & settings) const +void CreatingSetStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); - for (const auto & set : subqueries_for_sets) + settings.out << prefix; + if (subquery_for_set.set) + settings.out << "Set: "; + else if (subquery_for_set.join) + settings.out << "Join: "; + + settings.out << description << '\n'; +} + +CreatingSetsStep::CreatingSetsStep(DataStreams input_streams_) +{ + if (input_streams_.empty()) + throw Exception("CreatingSetsStep cannot be created with no inputs", ErrorCodes::LOGICAL_ERROR); + + input_streams = std::move(input_streams_); + output_stream = input_streams.front(); + + for (size_t i = 1; i < input_streams.size(); ++i) + assertBlocksHaveEqualStructure(output_stream->header, input_streams[i].header, "CreatingSets"); +} + +QueryPipelinePtr CreatingSetsStep::updatePipeline(QueryPipelines pipelines) +{ + if (pipelines.empty()) + throw Exception("CreatingSetsStep cannot be created with no inputs", ErrorCodes::LOGICAL_ERROR); + + auto main_pipeline = std::move(pipelines.front()); + if (pipelines.size() == 1) + return main_pipeline; + + std::swap(pipelines.front(), pipelines.back()); + pipelines.pop_back(); + + QueryPipeline delayed_pipeline; + if (pipelines.size() > 1) { - settings.out << prefix; - if (set.second.set) - settings.out << "Set: "; - else if (set.second.join) - settings.out << "Join: "; - - settings.out << set.first << '\n'; + QueryPipelineProcessorsCollector collector(delayed_pipeline, this); + delayed_pipeline = QueryPipeline::unitePipelines(std::move(pipelines), output_stream->header); + processors = collector.detachProcessors(); } + else + delayed_pipeline = std::move(*pipelines.front()); + + QueryPipelineProcessorsCollector collector(*main_pipeline, this); + main_pipeline->addDelayedPipeline(std::move(delayed_pipeline)); + auto added_processors = collector.detachProcessors(); + processors.insert(processors.end(), added_processors.begin(), added_processors.end()); + + return main_pipeline; +} + +void CreatingSetsStep::describePipeline(FormatSettings & settings) const +{ + IQueryPlanStep::describePipeline(processors, settings); } } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index 4ba4863c043..f2351c91518 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -7,12 +7,14 @@ namespace DB { /// Creates sets for subqueries and JOIN. See CreatingSetsTransform. -class CreatingSetsStep : public ITransformingStep +class CreatingSetStep : public ITransformingStep { public: - CreatingSetsStep( + CreatingSetStep( const DataStream & input_stream_, - SubqueriesForSets subqueries_for_sets_, + Block header, + String description_, + SubqueryForSet subquery_for_set_, SizeLimits network_transfer_limits_, const Context & context_); @@ -23,9 +25,25 @@ public: void describeActions(FormatSettings & settings) const override; private: - SubqueriesForSets subqueries_for_sets; + String description; + SubqueryForSet subquery_for_set; SizeLimits network_transfer_limits; const Context & context; }; +class CreatingSetsStep : public IQueryPlanStep +{ +public: + CreatingSetsStep(DataStreams input_streams_); + + String getName() const override { return "CreatingSets"; } + + QueryPipelinePtr updatePipeline(QueryPipelines pipelines) override; + + void describePipeline(FormatSettings & settings) const override; + +private: + Processors processors; +}; + } From 18bb5f026ae4c453c7e293828e047867e29c50bd Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Tue, 15 Sep 2020 16:37:12 +0300 Subject: [PATCH 072/390] changes after review --- docs/ru/operations/settings/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index da1c56e3daf..4854e39d96d 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1618,7 +1618,7 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; ## low\_cardinality\_max\_dictionary\_size {#low_cardinality_max_dictionary_size} -Задает максимальную длину строк в общем глобальном словаре для типа данных `LowCardinality`, который может быть записан в файловую систему хранилища. Настройка предоствращает проблемы с оперативной памятью в случае неограниченного увеличения словаря. Все данные, которые не могут быть закодированы из-за ограничения максимального размера словаря, ClickHouse записывает обычным способом. +Задает максимальный размер общего глобального словаря (в строках) для типа данных `LowCardinality`, который может быть записан в файловую систему хранилища. Настройка предотвращает проблемы с оперативной памятью в случае неограниченного увеличения словаря. Все данные, которые не могут быть закодированы из-за ограничения максимального размера словаря, ClickHouse записывает обычным способом. Допустимые значения: @@ -1628,7 +1628,7 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; ## low\_cardinality\_use\_single\_dictionary\_for\_part {#low_cardinality_use_single_dictionary_for_part} -Включает или выключает использование единого словаря для частей данных. +Включает или выключает использование единого словаря для куска (парта). По умолчанию сервер ClickHouse следит за размером словарей, и если словарь переполняется, сервер создает следующий. Чтобы запретить создание нескольких словарей, задайте настройку `low_cardinality_use_single_dictionary_for_part = 1`. From 7c0cafab0a439cd7e2164c33ac3d1a756cc21db7 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Tue, 15 Sep 2020 16:58:39 +0300 Subject: [PATCH 073/390] resolving conflict --- .../functions/type-conversion-functions.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 3b70f0d6577..1c09e4749f6 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -513,6 +513,97 @@ SELECT parseDateTimeBestEffort('10 20:19') - [toDate](#todate) - [toDateTime](#todatetime) +## toUnixTimestamp64Milli +## toUnixTimestamp64Micro +## toUnixTimestamp64Nano + +Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды. +Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение - это временная метка в UTC, а не в часовом поясе `DateTime64`. + +**Синтаксис** + +``` sql +toUnixTimestamp64Milli(value) +``` + +**Параметры** + +- `value` — значение `DateTime64` с любой точностью. + +**Возвращаемое значение** + +- Значение `value`, преобразованное в тип данных `Int64`. + +**Примеры** + +Запрос: + +``` sql +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT toUnixTimestamp64Milli(dt64) +``` + +Ответ: + +``` text +┌─toUnixTimestamp64Milli(dt64)─┐ +│ 1568650812345 │ +└──────────────────────────────┘ +``` + +Запрос: + +``` sql +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 +SELECT toUnixTimestamp64Nano(dt64) +``` + +Ответ: + +``` text +┌─toUnixTimestamp64Nano(dt64)─┐ +│ 1568650812345678000 │ +└─────────────────────────────┘ +``` + +## fromUnixTimestamp64Milli +## fromUnixTimestamp64Micro +## fromUnixTimestamp64Nano + +Преобразует значение `Int64` в значение `DateTime64` с фиксированной точностью менее одной секунды и дополнительным часовым поясом. Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что входное значение обрабатывается как метка времени UTC, а не метка времени в заданном (или неявном) часовом поясе. + +**Синтаксис** + +``` sql +fromUnixTimestamp64Milli(value [, ti]) +``` + +**Параметры** + +- `value` — значение типы `Int64` с любой точностью. +- `timezone` — (не обязательный параметр) часовой пояс в формате `String` для возвращаемого результата. + +**Возвращаемое значение** + +- Значение `value`, преобразованное в тип данных `DateTime64`. + +**Пример** + +Запрос: + +``` sql +WITH CAST(1234567891011, 'Int64') AS i64 +SELECT fromUnixTimestamp64Milli(i64, 'UTC') +``` + +Ответ: + +``` text +┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐ +│ 2009-02-13 23:31:31.011 │ +└──────────────────────────────────────┘ +``` + ## toLowCardinality {#tolowcardinality} Преобразует входные данные в версию [LowCardianlity](../data-types/lowcardinality.md) того же типа данных. From 7465e00163a5e02fa6928513a6cae89023dcab5d Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 15 Sep 2020 17:22:32 +0300 Subject: [PATCH 074/390] Optimized marks selection algorithm for continuous marks ranges --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 94 +++++++------------ 1 file changed, 35 insertions(+), 59 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index e780ebda111..f2010b4e34e 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1498,79 +1498,55 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( } else { - // Do inclusion search, where we only look for one range + // For the case of one continuous range of keys we use binary search algorithm + + LOG_TRACE(log, "Running binary search on index range for part {} ({} marks)", part->name, marks_count); size_t steps = 0; - auto find_leaf = [&](bool left) -> std::optional + MarkRange result_range; + + size_t searched_left = 0; + size_t searched_right = marks_count; + + while (searched_left + 1 < searched_right) { - std::vector stack = {}; - - MarkRange range = {0, marks_count}; - - steps++; - + const size_t middle = (searched_left + searched_right) / 2; + MarkRange range(0, middle); if (may_be_true_in_range(range)) - stack.emplace_back(range.begin, range.end); + searched_right = middle; + else + searched_left = middle; + ++steps; + } + result_range.begin = searched_left; + LOG_TRACE(log, "Found (LEFT) boundary mark: {}", searched_left); - while (!stack.empty()) - { - range = stack.back(); - stack.pop_back(); + searched_right = marks_count; + while (searched_left + 1 < searched_right) + { + const size_t middle = (searched_left + searched_right) / 2; + MarkRange range(middle, marks_count); + if (may_be_true_in_range(range)) + searched_left = middle; + else + searched_right = middle; + ++steps; + } + result_range.end = searched_right; + LOG_TRACE(log, "Found (RIGHT) boundary mark: {}", searched_right); - if (range.end == range.begin + 1) - { - if (left) - return range.begin; - else - return range.end; - } - else - { - std::vector check_order = {}; - MarkRange left_range = {range.begin, (range.begin + range.end) / 2}; - MarkRange right_range = {(range.begin + range.end) / 2, range.end}; + if (may_be_true_in_range(result_range)) + res.emplace_back(std::move(result_range)); - if (left) - { - check_order.emplace_back(left_range.begin, left_range.end); - check_order.emplace_back(right_range.begin, right_range.end); - } - else - { - check_order.emplace_back(right_range.begin, right_range.end); - check_order.emplace_back(left_range.begin, left_range.end); - } - - steps++; - - if (may_be_true_in_range(check_order[0])) - { - stack.emplace_back(check_order[0].begin, check_order[0].end); - continue; - } - - if (may_be_true_in_range(check_order[1])) - stack.emplace_back(check_order[1].begin, check_order[1].end); - else - break; // No mark range would suffice - } - } - - return std::nullopt; - }; - - auto left_leaf = find_leaf(true); - if (left_leaf) - res.emplace_back(left_leaf.value(), find_leaf(false).value()); - - LOG_TRACE(log, "Used optimized inclusion search over index for part {} with {} steps", part->name, steps); + LOG_TRACE(log, "Found {} range in {} steps", res.empty() ? "empty" : "continuous", steps); } return res; } + MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( MergeTreeIndexPtr index_helper, MergeTreeIndexConditionPtr condition, From a792850ecd69934e4294d7b65ba1a14459e9de1f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 15 Sep 2020 18:05:42 +0300 Subject: [PATCH 075/390] Update ThreadPool.cpp --- src/Common/ThreadPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 93aa6be8d9a..737826e3027 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -279,7 +279,7 @@ void GlobalThreadPool::initialize(size_t max_threads) { if (the_instance) { - throw Exception(LOGICAL_ERROR, + throw Exception(ErrorCodes::LOGICAL_ERROR, "The global thread pool is initialized twice"); } From 5afb19faf1893113e978a330c42418a0cc0f3fba Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 15 Sep 2020 19:58:09 +0300 Subject: [PATCH 076/390] Update ThreadPool.cpp --- src/Common/ThreadPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 737826e3027..cb8a7669eef 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -279,7 +279,7 @@ void GlobalThreadPool::initialize(size_t max_threads) { if (the_instance) { - throw Exception(ErrorCodes::LOGICAL_ERROR, + throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "The global thread pool is initialized twice"); } From 118a8a513e6eea1b24d5f5e4464c0e746c597046 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Sep 2020 20:13:13 +0300 Subject: [PATCH 077/390] Update mutations interpreter. --- src/Interpreters/InterpreterSelectQuery.cpp | 29 +--------- .../InterpreterSelectWithUnionQuery.cpp | 7 ++- src/Interpreters/MutationsInterpreter.cpp | 50 ++++++++++------- src/Interpreters/MutationsInterpreter.h | 5 +- src/Interpreters/SubqueryForSet.cpp | 5 ++ src/Interpreters/SubqueryForSet.h | 5 ++ src/Processors/QueryPipeline.cpp | 55 +------------------ src/Processors/QueryPipeline.h | 6 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 37 ++++++++++++- src/Processors/QueryPlan/CreatingSetsStep.h | 8 ++- src/Processors/QueryPlan/QueryPlan.cpp | 14 +++-- src/Processors/QueryPlan/QueryPlan.h | 4 +- .../QueryPlan/ReadFromPreparedSource.cpp | 3 +- .../QueryPlan/ReadFromPreparedSource.h | 2 +- src/Processors/Transforms/JoiningTransform.h | 2 +- 15 files changed, 111 insertions(+), 121 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6f1be43b3a1..6d4362baeca 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1866,34 +1866,7 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_p return; SizeLimits limits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); - - std::vector plans; - DataStreams input_streams; - input_streams.emplace_back(query_plan.getCurrentDataStream()); - - for (auto & [description, set] : subqueries_for_sets) - { - auto plan = std::move(set.source); - std::string type = (set.join != nullptr) ? "JOIN" - : "subquery"; - - auto creating_set = std::make_unique( - plan->getCurrentDataStream(), - query_plan.getCurrentDataStream().header, - std::move(description), - std::move(set), - limits, - *context); - creating_set->setStepDescription("Create set for " + type); - plan->addStep(std::move(creating_set)); - - input_streams.emplace_back(plan->getCurrentDataStream()); - plans.emplace_back(std::move(*plan)); - } - - auto creating_sets = std::make_unique(std::move(input_streams)); - creating_sets->setStepDescription("Create sets before main query execution"); - query_plan.unitePlans(std::move(creating_sets), std::move(plans)); + addCreatingSetsStep(query_plan, std::move(subqueries_for_sets), limits, *context); } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 1e631ea538b..ba0ebfaaf27 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -183,13 +183,14 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) return; } - std::vector plans(num_plans); + std::vector> plans(num_plans); DataStreams data_streams(num_plans); for (size_t i = 0; i < num_plans; ++i) { - nested_interpreters[i]->buildQueryPlan(plans[i]); - data_streams[i] = plans[i].getCurrentDataStream(); + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + data_streams[i] = plans[i]->getCurrentDataStream(); } auto max_threads = context->getSettingsRef().max_threads; diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 089e3d1c23f..7f6f3441f5b 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -11,6 +11,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -19,6 +24,7 @@ #include #include #include +#include namespace DB @@ -524,10 +530,11 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()}; auto first_stage_header = interpreter.getSampleBlock(); - QueryPipeline pipeline; - pipeline.init(Pipe(std::make_shared(first_stage_header))); - addStreamsForLaterStages(stages_copy, pipeline); - updated_header = std::make_unique(pipeline.getHeader()); + QueryPlan plan; + auto source = std::make_shared(first_stage_header); + plan.addStep(std::make_unique(Pipe(std::move(source)))); + auto pipeline = addStreamsForLaterStages(stages_copy, plan); + updated_header = std::make_unique(pipeline->getHeader()); } /// Special step to recalculate affected indices and TTL expressions. @@ -656,7 +663,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & return select; } -void MutationsInterpreter::addStreamsForLaterStages(const std::vector & prepared_stages, QueryPipeline & pipeline) const +QueryPipelinePtr MutationsInterpreter::addStreamsForLaterStages(const std::vector & prepared_stages, QueryPlan & plan) const { for (size_t i_stage = 1; i_stage < prepared_stages.size(); ++i_stage) { @@ -668,18 +675,12 @@ void MutationsInterpreter::addStreamsForLaterStages(const std::vector & p if (i < stage.filter_column_names.size()) { /// Execute DELETEs. - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, step->actions(), stage.filter_column_names[i], false); - }); + plan.addStep(std::make_unique(plan.getCurrentDataStream(), step->actions(), stage.filter_column_names[i], false)); } else { /// Execute UPDATE or final projection. - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, step->actions()); - }); + plan.addStep(std::make_unique(plan.getCurrentDataStream(), step->actions())); } } @@ -689,14 +690,17 @@ void MutationsInterpreter::addStreamsForLaterStages(const std::vector & p const Settings & settings = context.getSettingsRef(); SizeLimits network_transfer_limits( settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); - pipeline.addCreatingSetsTransform(std::move(subqueries_for_sets), network_transfer_limits, context); + addCreatingSetsStep(plan, std::move(subqueries_for_sets), network_transfer_limits, context); } } - pipeline.addSimpleTransform([&](const Block & header) + auto pipeline = plan.buildQueryPipeline(); + pipeline->addSimpleTransform([&](const Block & header) { return std::make_shared(header); }); + + return pipeline; } void MutationsInterpreter::validate() @@ -718,8 +722,11 @@ void MutationsInterpreter::validate() } } - auto block_io = select_interpreter->execute(); - addStreamsForLaterStages(stages, block_io.pipeline); + QueryPlan plan; + select_interpreter->buildQueryPlan(plan); + addStreamsForLaterStages(stages, plan); + + auto pipeline = plan.buildQueryPipeline(); } BlockInputStreamPtr MutationsInterpreter::execute() @@ -727,10 +734,13 @@ BlockInputStreamPtr MutationsInterpreter::execute() if (!can_execute) throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR); - auto block_io = select_interpreter->execute(); - addStreamsForLaterStages(stages, block_io.pipeline); + QueryPlan plan; + select_interpreter->buildQueryPlan(plan); - auto result_stream = block_io.getInputStream(); + addStreamsForLaterStages(stages, plan); + + auto pipeline = plan.buildQueryPipeline(); + BlockInputStreamPtr result_stream = std::make_shared(std::move(*pipeline)); /// Sometimes we update just part of columns (for example UPDATE mutation) /// in this case we don't read sorting key, so just we don't check anything. diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 359ee1a3fd0..59d9e7657c3 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -13,7 +13,10 @@ namespace DB { class Context; +class QueryPlan; + class QueryPipeline; +using QueryPipelinePtr = std::unique_ptr; /// Return false if the data isn't going to be changed by mutations. bool isStorageTouchedByMutations( @@ -52,7 +55,7 @@ private: struct Stage; ASTPtr prepareInterpreterSelectQuery(std::vector &prepared_stages, bool dry_run); - void addStreamsForLaterStages(const std::vector & prepared_stages, QueryPipeline & pipeline) const; + QueryPipelinePtr addStreamsForLaterStages(const std::vector & prepared_stages, QueryPlan & plan) const; std::optional getStorageSortDescriptionIfPossible(const Block & header) const; diff --git a/src/Interpreters/SubqueryForSet.cpp b/src/Interpreters/SubqueryForSet.cpp index e944b76e71c..17ea813c545 100644 --- a/src/Interpreters/SubqueryForSet.cpp +++ b/src/Interpreters/SubqueryForSet.cpp @@ -8,6 +8,11 @@ namespace DB { +SubqueryForSet::SubqueryForSet() = default; +SubqueryForSet::~SubqueryForSet() = default; +SubqueryForSet::SubqueryForSet(SubqueryForSet &&) = default; +SubqueryForSet & SubqueryForSet::operator= (SubqueryForSet &&) = default; + void SubqueryForSet::makeSource(std::shared_ptr & interpreter, NamesWithAliases && joined_block_aliases_) { diff --git a/src/Interpreters/SubqueryForSet.h b/src/Interpreters/SubqueryForSet.h index b44f0f6cf8b..fd073500dc2 100644 --- a/src/Interpreters/SubqueryForSet.h +++ b/src/Interpreters/SubqueryForSet.h @@ -18,6 +18,11 @@ class QueryPlan; /// Information on what to do when executing a subquery in the [GLOBAL] IN/JOIN section. struct SubqueryForSet { + SubqueryForSet(); + ~SubqueryForSet(); + SubqueryForSet(SubqueryForSet &&); + SubqueryForSet & operator= (SubqueryForSet &&); + /// The source is obtained using the InterpreterSelectQuery subquery. std::unique_ptr source; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index c7ffe8b0c9b..0337f19c2e3 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -196,59 +196,6 @@ void QueryPipeline::addExtremesTransform() pipe.addTransform(std::move(transform), nullptr, port); } -void QueryPipeline::addCreatingSetsTransform(SubqueriesForSets subqueries_for_sets, const SizeLimits & network_transfer_limits, const Context & context) -{ - checkInitializedAndNotCompleted(); - - Pipes sources; - - for (auto & subquery : subqueries_for_sets) - { - if (subquery.second.source) - { - auto & source = sources.emplace_back(std::move(subquery.second.source)); - if (source.numOutputPorts() > 1) - source.addTransform(std::make_shared(source.getHeader(), source.numOutputPorts(), 1)); - - source.dropExtremes(); - - auto creating_sets = std::make_shared( - source.getHeader(), - getHeader(), - std::move(subquery.second), - network_transfer_limits, - context); - - InputPort * totals = nullptr; - if (source.getTotalsPort()) - totals = creating_sets->addTotalsPort(); - - source.addTransform(std::move(creating_sets), totals, nullptr); - } - } - - if (sources.empty()) - return; - - auto * collected_processors = pipe.collected_processors; - - /// We unite all sources together. - /// Set collected_processors to attach all newly-added processors to current query plan step. - auto source = Pipe::unitePipes(std::move(sources), collected_processors); - if (source.numOutputPorts() > 1) - source.addTransform(std::make_shared(source.getHeader(), source.numOutputPorts(), 1)); - source.collected_processors = nullptr; - - resize(1); - - Pipes pipes; - pipes.emplace_back(std::move(source)); - pipes.emplace_back(std::move(pipe)); - pipe = Pipe::unitePipes(std::move(pipes), collected_processors); - - pipe.addTransform(std::make_shared(getHeader(), 2)); -} - void QueryPipeline::setOutputFormat(ProcessorPtr output) { checkInitializedAndNotCompleted(); @@ -315,7 +262,7 @@ QueryPipeline QueryPipeline::unitePipelines( return pipeline; } -void QueryPipeline::addDelayedPipeline(QueryPipeline pipeline) +void QueryPipeline::addDelayingPipeline(QueryPipeline pipeline) { pipeline.resize(1); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 06c67c897be..4be7bbfd573 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -55,8 +55,6 @@ public: void addTotalsHavingTransform(ProcessorPtr transform); /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. void addExtremesTransform(); - /// Adds transform which creates sets. It will be executed before reading any data from input ports. - void addCreatingSetsTransform(SubqueriesForSets subqueries_for_sets, const SizeLimits & network_transfer_limits, const Context & context); /// Resize pipeline to single output and add IOutputFormat. Pipeline will be completed after this transformation. void setOutputFormat(ProcessorPtr output); /// Get current OutputFormat. @@ -87,7 +85,9 @@ public: size_t max_threads_limit = 0, Processors * collected_processors = nullptr); - void addDelayedPipeline(QueryPipeline); + /// Add other pipeline and execute it before current one. + /// Pipeline must have same header. + void addDelayingPipeline(QueryPipeline pipeline); PipelineExecutorPtr execute(); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 9a4c11f9222..f63b657c029 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -97,7 +97,7 @@ QueryPipelinePtr CreatingSetsStep::updatePipeline(QueryPipelines pipelines) delayed_pipeline = std::move(*pipelines.front()); QueryPipelineProcessorsCollector collector(*main_pipeline, this); - main_pipeline->addDelayedPipeline(std::move(delayed_pipeline)); + main_pipeline->addDelayingPipeline(std::move(delayed_pipeline)); auto added_processors = collector.detachProcessors(); processors.insert(processors.end(), added_processors.begin(), added_processors.end()); @@ -109,4 +109,39 @@ void CreatingSetsStep::describePipeline(FormatSettings & settings) const IQueryPlanStep::describePipeline(processors, settings); } +void addCreatingSetsStep( + QueryPlan & query_plan, SubqueriesForSets subqueries_for_sets, const SizeLimits & limits, const Context & context) +{ + DataStreams input_streams; + input_streams.emplace_back(query_plan.getCurrentDataStream()); + + std::vector> plans; + plans.emplace_back(std::make_unique(std::move(query_plan))); + query_plan = QueryPlan(); + + for (auto & [description, set] : subqueries_for_sets) + { + auto plan = std::move(set.source); + std::string type = (set.join != nullptr) ? "JOIN" + : "subquery"; + + auto creating_set = std::make_unique( + plan->getCurrentDataStream(), + input_streams.front().header, + std::move(description), + std::move(set), + limits, + context); + creating_set->setStepDescription("Create set for " + type); + plan->addStep(std::move(creating_set)); + + input_streams.emplace_back(plan->getCurrentDataStream()); + plans.emplace_back(std::move(plan)); + } + + auto creating_sets = std::make_unique(std::move(input_streams)); + creating_sets->setStepDescription("Create sets before main query execution"); + query_plan.unitePlans(std::move(creating_sets), std::move(plans)); +} + } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index f2351c91518..ec13ab2052e 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -18,7 +18,7 @@ public: SizeLimits network_transfer_limits_, const Context & context_); - String getName() const override { return "CreatingSets"; } + String getName() const override { return "CreatingSet"; } void transformPipeline(QueryPipeline & pipeline) override; @@ -46,4 +46,10 @@ private: Processors processors; }; +void addCreatingSetsStep( + QueryPlan & query_plan, + SubqueriesForSets subqueries_for_sets, + const SizeLimits & limits, + const Context & context); + } diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 31b9de2fcee..74781f4b5d9 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -26,6 +26,8 @@ namespace ErrorCodes QueryPlan::QueryPlan() = default; QueryPlan::~QueryPlan() = default; +QueryPlan::QueryPlan(QueryPlan &&) = default; +QueryPlan & QueryPlan::operator=(QueryPlan &&) = default; void QueryPlan::checkInitialized() const { @@ -51,7 +53,7 @@ const DataStream & QueryPlan::getCurrentDataStream() const return root->step->getOutputStream(); } -void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vector plans) +void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vector> plans) { if (isInitialized()) throw Exception("Cannot unite plans because current QueryPlan is already initialized", @@ -70,7 +72,7 @@ void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vector plans) for (size_t i = 0; i < num_inputs; ++i) { const auto & step_header = inputs[i].header; - const auto & plan_header = plans[i].getCurrentDataStream().header; + const auto & plan_header = plans[i]->getCurrentDataStream().header; if (!blocksHaveEqualStructure(step_header, plan_header)) throw Exception("Cannot unite QueryPlans using " + step->getName() + " because " "it has incompatible header with plan " + root->step->getName() + " " @@ -79,19 +81,19 @@ void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vector plans) } for (auto & plan : plans) - nodes.splice(nodes.end(), std::move(plan.nodes)); + nodes.splice(nodes.end(), std::move(plan->nodes)); nodes.emplace_back(Node{.step = std::move(step)}); root = &nodes.back(); for (auto & plan : plans) - root->children.emplace_back(plan.root); + root->children.emplace_back(plan->root); for (auto & plan : plans) { - max_threads = std::max(max_threads, plan.max_threads); + max_threads = std::max(max_threads, plan->max_threads); interpreter_context.insert(interpreter_context.end(), - plan.interpreter_context.begin(), plan.interpreter_context.end()); + plan->interpreter_context.begin(), plan->interpreter_context.end()); } } diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 7ce8d9426c4..6296eac7502 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -25,8 +25,10 @@ class QueryPlan public: QueryPlan(); ~QueryPlan(); + QueryPlan(QueryPlan &&); + QueryPlan & operator=(QueryPlan &&); - void unitePlans(QueryPlanStepPtr step, std::vector plans); + void unitePlans(QueryPlanStepPtr step, std::vector> plans); void addStep(QueryPlanStepPtr step); bool isInitialized() const { return root != nullptr; } /// Tree is not empty diff --git a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp index 6f0d1693ce0..979b4101046 100644 --- a/src/Processors/QueryPlan/ReadFromPreparedSource.cpp +++ b/src/Processors/QueryPlan/ReadFromPreparedSource.cpp @@ -14,7 +14,8 @@ ReadFromPreparedSource::ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_); + explicit ReadFromPreparedSource(Pipe pipe_, std::shared_ptr context_ = nullptr); String getName() const override { return "ReadNothing"; } diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h index c00ac5b83dd..15a203635e2 100644 --- a/src/Processors/Transforms/JoiningTransform.h +++ b/src/Processors/Transforms/JoiningTransform.h @@ -14,7 +14,7 @@ public: JoiningTransform(Block input_header, JoinPtr join_, bool on_totals_ = false, bool default_totals_ = false); - String getName() const override { return "InflatingExpressionTransform"; } + String getName() const override { return "JoiningTransform"; } static Block transformHeader(Block header, const JoinPtr & join); From 13edfeea1fa7c0e9f0a5b840240933b4854a6ee0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Sep 2020 20:23:49 +0300 Subject: [PATCH 078/390] Fix build. --- src/Storages/IStorage.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index dc7c684d5b4..c68b910494e 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -53,6 +53,9 @@ class QueryPlan; class StoragePolicy; using StoragePolicyPtr = std::shared_ptr; +struct StreamLocalLimits; +class EnabledQuota; + struct ColumnSize { size_t marks = 0; From bd1a7c69bae76b42eac8b59b469a7b30eb1d3714 Mon Sep 17 00:00:00 2001 From: myrrc Date: Tue, 15 Sep 2020 22:32:42 +0300 Subject: [PATCH 079/390] rewrote code from bash to python --- CMakeLists.txt | 12 ++- cmake_files_footer.md | 123 +++++++++++++++++++++++ cmake_files_header.md | 125 +---------------------- cmake_flags_and_output.md | 194 +++++++++++++++--------------------- cmake_flags_generator.py | 104 +++++++++++++++++++ cmake_flags_md_generator.sh | 38 ------- 6 files changed, 320 insertions(+), 276 deletions(-) create mode 100644 cmake_files_footer.md create mode 100755 cmake_flags_generator.py delete mode 100755 cmake_flags_md_generator.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index cf97b2c40ff..09a4500e314 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,9 +29,8 @@ endforeach() project(ClickHouse) option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION - "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" - ON -) + "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) + if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) else() @@ -58,7 +57,11 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -option(ENABLE_IPO "Enable full link time optimization (it's usually impractical; see also ENABLE_THINLTO)" OFF) # need cmake 3.9+ +# Need cmake 3.9+ +# Usually impractical. +# See also ENABLE_THINLTO +option(ENABLE_IPO "Enable full link time optimization") + if(ENABLE_IPO) cmake_policy(SET CMP0069 NEW) include(CheckIPOSupported) @@ -90,6 +93,7 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) + if (NOT MAKE_STATIC_LIBRARIES) option (SPLIT_SHARED_LIBRARIES "DEV ONLY. Keep all internal libs as separate .so for faster linking" OFF) option (CLICKHOUSE_SPLIT_BINARY "Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... )" OFF) diff --git a/cmake_files_footer.md b/cmake_files_footer.md new file mode 100644 index 00000000000..b67f79b9b65 --- /dev/null +++ b/cmake_files_footer.md @@ -0,0 +1,123 @@ +## Developer's guide for adding new CMake options + +### Don't be obvious. Be informative. + +Bad: +```cmake +option (ENABLE_TESTS "Enables testing" OFF) +``` + +This description is quite useless as is neither gives the viewer any additional information nor explains the option +purpose. If the option's name is quite self-descriptive, prefer the empty description. + +Better: + +```cmake +option(ENABLE_TESTS OFF) +``` + +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above +the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). +The comment is parsed into a separate column (see below). + +Even better (default off value is omitted): + +```cmake +# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). +# see tests/CMakeLists.txt for implementation detail. +option(ENABLE_GTEST_TESTS) +``` + +### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. + +Suppose you have an option that may strip debug symbols from the ClickHouse's part. +This can speed up the linking process, but produces a binary that cannot be debugged. +In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. +Also, such options should be disabled if applies. + +Bad: +```cmake +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions. + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() + +``` +Better: + +```cmake +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions." + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + message(WARNING "Not generating debugger info for ClickHouse functions") + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() +``` + +### In the option's description, explain WHAT the option does rather than WHY it does something. + +The WHY explanation should be placed in the comment. +You may find that the option's name is self-descriptive. + +Bad: + +```cmake +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better: + +```cmake +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +### Don't assume other developers know as much as you do. + +In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to +the tool's docs. It won't take much of your time. + +Bad: + +```cmake +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better (combined with the above hint): + +```cmake +# https://clang.llvm.org/docs/ThinLTO.html +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO ON). +``` + +Other example, bad: + +```cmake +option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) +``` + +Better: + +```cmake +# https://github.com/include-what-you-use/include-what-you-use +option (USE_INCLUDE_WHAT_YOU_USE) +``` + +### Prefer consistent default values. + +CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. +Prefer the `ON/OFF` values, if possible. + + + diff --git a/cmake_files_header.md b/cmake_files_header.md index e1fa8ad29fe..333a1bf5f30 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -1,131 +1,14 @@ -# CMake flags +# All about CMake in ClickHouse -## Developer's guide for adding new CMake options - -### Don't be obvious. Be informative. - -Bad: -```cmake -option (ENABLE_TESTS "Enables testing" OFF) -``` - -This description is quite useless as is neither gives the viewer any additional information nor explains the option -purpose. If the option's name is quite self-descriptive, prefer the empty description. - -Better: - -```cmake -option(ENABLE_TESTS OFF) -``` - -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above -the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). -The comment is parsed into a separate column (see below). - -Even better (default off value is omitted): - -```cmake -# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). -# see tests/CMakeLists.txt for implementation detail. -option(ENABLE_GTEST_TESTS) -``` - -### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. - -Suppose you have an option that may strip debug symbols from the ClickHouse's part. -This can speed up the linking process, but produces a binary that cannot be debugged. -In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. -Also, such options should be disabled if applies. - -Bad: -```cmake -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() - -``` -Better: - -```cmake -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions." - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -### In the option's description, explain WHAT the option does rather than WHY it does something. - -The WHY explanation should be placed in the comment. -You may find that the option's name is self-descriptive. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better: - -```cmake -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). -``` - -### Don't assume other developers know as much as you do. - -In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to -the tool's docs. It won't take much of your time. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better (combined with the above hint): - -```cmake -# https://clang.llvm.org/docs/ThinLTO.html -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). -``` - -Other example, bad: - -```cmake -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) -``` - -Better: - -```cmake -# https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE) -``` - -### Prefer consistent default values. - -CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. -Prefer the `ON/OFF` values, if possible. +## How ClickHouse uses CMake +TODO describe separate cmake files for contrib + arch-dependent ones + options finding. ## List of CMake flags * This list is auto-generated by [this bash script](bash.sh). * The flag name is a link to its position in the code. +* If an option's default valus is itself an option, it's also a link to its position in this list. | Name | Default value | Description | Comment | |------|---------------|-------------|---------| diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 65ad40dc916..11f5ed36496 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -1,5 +1,85 @@ -# CMake flags +# All about CMake in ClickHouse +## How ClickHouse uses CMake + +TODO describe separate cmake files for contrib + arch-dependent ones + options finding. + +## List of CMake flags + +* This list is auto-generated by [this bash script](bash.sh). +* The flag name is a link to its position in the code. +* If an option's default valus is itself an option, it's also a link to its position in this list. + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| (`ADD_GDB_INDEX_FOR_GOLD`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L181] | `0` | Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| (`ARCH_NATIVE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L215] | `0` | Enable -march=native compiler flag | | +| (`CLICKHOUSE_SPLIT_BINARY`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L98] | `OFF` | Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... ) | | +| (`COMPILER_PIPE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L203] | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| (`ENABLE_FUZZING`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L113] | `OFF` | Enables fuzzing instrumentation | | +| (`ENABLE_IPO`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62] | `OFF` | Enable full link time optimization | | +| (`ENABLE_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L308] | `ON` | Enable all libraries (Global default switch) | | +| (`ENABLE_TESTS`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L145] | `ON` | Enables tests | | +| (`ENABLE_THINLTO`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L61] | `ON` | Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers. | Need cmake 3.9+ +Usually impractical. +See also ENABLE_THINLTO + | +| (`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30] | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | +| (`GLIBC_COMPATIBILITY`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L148] | `ON` | Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY. | | +| (`MAKE_STATIC_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to make shared libraries | | +| (`SPLIT_SHARED_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L97] | `OFF` | DEV ONLY. Keep all internal libs as separate .so for faster linking | | +| (`UNBUNDLED`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L147] | `OFF` | Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages. | | +| (`USE_INCLUDE_WHAT_YOU_USE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L364] | `OFF` | Use 'include-what-you-use' tool | | +| (`USE_STATIC_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L93] | `ON` | Set to FALSE to use shared libraries | | +| (`WERROR`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L319] | `OFF` | Enable -Werror compiler option | | +| (`WERROR`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L319] | `ON` | Enable -Werror compiler option | | +| (`WITH_COVERAGE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L117] | `0` | Build with coverage. | || (`WEVERYTHING`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20] | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | || (`PARALLEL_COMPILE_JOBS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8] | `""` | Define the maximum number of concurrent compilation jobs | | +| (`PARALLEL_LINK_JOBS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21] | `""` | Define the maximum number of concurrent link jobs | || (`ENABLE_CLANG_TIDY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2] | `OFF` | Use 'clang-tidy' static analyzer if present | || (`SANITIZE`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0] | `""` | Enable sanitizer: address, memory, thread, undefined | || (`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use bundled library ${LIB_NAME} instead of system | || (`FUZZER`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0] | `OFF` | Enable fuzzer: libfuzzer | || (`LINKER_NAME`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42] | `OFF` | Linker name or full path | || (`ENABLE_SSL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ssl | | +| (`USE_INTERNAL_SSL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system *ssl library instead of bundled | || (`ENABLE_GPERF`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use gperf function hash generator tool | || (`USE_INTERNAL_SPARSEHASH_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0] | `ON` | Set to FALSE to use system sparsehash library instead of bundled | || (`ENABLE_BASE64`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable base64 | || (`USE_INTERNAL_RE2_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system re2 library instead of bundled [slower] | || (`ENABLE_H3`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable H3 | | +| (`USE_INTERNAL_H3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2] | `ON` | Set to FALSE to use system h3 library instead of bundled | || (`ENABLE_OPENCL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable OpenCL support | || (`USE_INTERNAL_LIBCXX_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3] | (`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`)[#use_internal_libcxx_library_default] | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | +| (`USE_LIBCXX`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use libc++ and libc++abi instead of libstdc++ | || (`ENABLE_MYSQL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable MySQL | | +| (`ENABLE_MYSQL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1] | `FALSE` | Enable MySQL | | +| (`USE_INTERNAL_MYSQL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system mysqlclient library instead of bundled | || (`USE_INTERNAL_ZLIB_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system zlib library instead of bundled | || (`USE_SIMDJSON`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use simdjson | || (`USE_INTERNAL_SNAPPY_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system snappy library instead of bundled | | +| (`USE_SNAPPY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable support of snappy library | || (`USE_SENTRY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use Sentry | || (`USE_INTERNAL_ZSTD_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system zstd library instead of bundled | || (`ENABLE_LDAP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable LDAP | | +| (`USE_INTERNAL_LDAP_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system *LDAP library instead of bundled | || (`ENABLE_CURL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable curl | | +| (`USE_INTERNAL_CURL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use internal curl library | || (`ENABLE_ODBC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ODBC library | | +| (`USE_INTERNAL_ODBC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use internal ODBC library | || (`USE_INTERNAL_LIBXML2_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system libxml2 library instead of bundled | || (`ENABLE_STATS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enalbe StatsLib library | || (`ENABLE_AVRO`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Avro | | +| (`USE_INTERNAL_AVRO_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3] | `ON` | Set to FALSE to use system avro library instead of bundled | || (`ENABLE_GTEST_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable gtest library | | +| (`USE_INTERNAL_GTEST_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system Google Test instead of bundled | || (`ENABLE_AMQPCPP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enalbe AMQP-CPP | || (`ENABLE_ICU`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ICU | | +| (`ENABLE_ICU`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1] | `0` | Enable ICU | | +| (`USE_INTERNAL_ICU_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system ICU library instead of bundled | || (`ENABLE_CCACHE`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6] | (`ENABLE_CCACHE_BY_DEFAULT`)[#enable_ccache_by_default] | Speedup re-compilations using ccache | || (`ENABLE_PARQUET`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable parquet | | +| (`USE_INTERNAL_PARQUET_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system parquet library instead of bundled | || (`USE_UNWIND`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable libunwind (better stacktraces) | || (`ENABLE_BROTLI`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable brotli | | +| (`USE_INTERNAL_BROTLI_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to use system libbrotli library instead of bundled | | +| (`USE_INTERNAL_BROTLI_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3] | `ON` | Set to FALSE to use system libbrotli library instead of bundled | || (`ENABLE_MSGPACK`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable msgpack library | | +| (`USE_INTERNAL_MSGPACK_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system msgpack library instead of bundled | || (`ENABLE_EMBEDDED_COMPILER`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| (`LLVM_HAS_RTTI`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39] | `ON` | Enable if LLVM was build with RTTI enabled | | +| (`USE_INTERNAL_LLVM_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use bundled or system LLVM library. | || (`ENABLE_HDFS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable HDFS | | +| (`USE_INTERNAL_HDFS3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2] | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | || (`ENABLE_GSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable gsasl library | | +| (`USE_INTERNAL_LIBGSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to use system libgsasl library instead of bundled | | +| (`USE_INTERNAL_LIBGSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3] | `ON` | Set to FALSE to use system libgsasl library instead of bundled | || (`ENABLE_FASTOPS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable fast vectorized mathematical functions library by Mikhail Parakhin | || (`ENABLE_RDKAFKA`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable kafka | | +| (`USE_INTERNAL_RDKAFKA_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system librdkafka instead of the bundled | || (`ENABLE_ORC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ORC | | +| (`USE_INTERNAL_ORC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3] | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | || (`ENABLE_CASSANDRA`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Cassandra | || (`ENABLE_S3`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable S3 | | +| (`USE_INTERNAL_AWS_S3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2] | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | || (`ENABLE_RAPIDJSON`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use rapidjson | | +| (`USE_INTERNAL_RAPIDJSON_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system rapidjson library instead of bundled | || (`USE_INTERNAL_POCO_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0] | `ON` | Use internal Poco library | || (`ENABLE_GRPC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use gRPC | | +| (`USE_INTERNAL_GRPC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | || (`ENABLE_PROTOBUF`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable protobuf | | +| (`USE_INTERNAL_PROTOBUF_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system protobuf instead of bundled | || (`ENABLE_CAPNP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Cap'n Proto | | +| (`USE_INTERNAL_CAPNP_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system capnproto library instead of bundled | || (`ENABLE_MULTITARGET_CODE`)[https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L117] | `ON` | | | +| (`STRIP_DEBUG_SYMBOLS_FUNCTIONS`)[https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64] | (`STRIP_DSF_DEFAULT`)[#strip_dsf_default] | Do not generate debugger info for ClickHouse functions. + Provides faster linking and lower binary size. + Tradeoff is the inability to debug some source files with e.g. gdb + (empty stack frames and no local variables). | || (`ENABLE_CLICKHOUSE_ALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8] | `ON` | Enable all tools | | +| (`ENABLE_CLICKHOUSE_BENCHMARK`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-benchmark | | +| (`ENABLE_CLICKHOUSE_CLIENT`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-client | | +| (`ENABLE_CLICKHOUSE_COMPRESSOR`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-compressor | | +| (`ENABLE_CLICKHOUSE_COPIER`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-copier | | +| (`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-extract-from-config | | +| (`ENABLE_CLICKHOUSE_FORMAT`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-format | | +| (`ENABLE_CLICKHOUSE_INSTALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-install | | +| (`ENABLE_CLICKHOUSE_INSTALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21] | `OFF` | Enable clickhouse-install | | +| (`ENABLE_CLICKHOUSE_LOCAL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L11] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-local | | +| (`ENABLE_CLICKHOUSE_OBFUSCATOR`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-obfuscator | | +| (`ENABLE_CLICKHOUSE_ODBC_BRIDGE`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-odbc-bridge | | +| (`ENABLE_CLICKHOUSE_SERVER`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-server | | ## Developer's guide for adding new CMake options ### Don't be obvious. Be informative. @@ -122,116 +202,4 @@ CMake allows you to pass a plethora of values representing boolean `true/false`, Prefer the `ON/OFF` values, if possible. -## List of CMake flags -* This list is auto-generated by [this bash script](cmake_flags_md_generator.sh). -* The flag name is a link to its position in the code. - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ENABLE_IPO`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `` | Enable full link time optimization | | -| [`USE_STATIC_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Set to FALSE to use shared libraries | | -| [`MAKE_STATIC_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to make shared libraries | | -| [`SPLIT_SHARED_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | DEV ONLY. Keep all internal libs as separate .so for faster linking | | -| [`CLICKHOUSE_SPLIT_BINARY`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... ) | | -| [`ENABLE_FUZZING`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Enables fuzzing instrumentation | | -| [`ENABLE_TESTS`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enables tests | | -| [`GLIBC_COMPATIBILITY`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY. | | -| [`ADD_GDB_INDEX_FOR_GOLD`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| [`COMPILER_PIPE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | -| [`ARCH_NATIVE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Enable -march=native compiler flag | | -| [`WITH_COVERAGE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `0` | Build with coverage. | | -| [`ENABLE_THINLTO`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers. | | -| [`ENABLE_LIBRARIES`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable all libraries (Global default switch) | | -| [`UNBUNDLED`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages. | | -| [`WERROR`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Enable -Werror compiler option | | -| [`WERROR`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `ON` | Enable -Werror compiler option | | -| [`USE_INCLUDE_WHAT_YOU_USE`](http://github.com/clickhouse/clickhouse/blob/master//CMakeLists.txt) | `OFF` | Use 'include-what-you-use' tool | | -| [`ENABLE_CLANG_TIDY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake) | `OFF` | Use 'clang-tidy' static analyzer if present | | -| [`USE_INTERNAL_`](http://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake) | `${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ${NOT_UNBUNDLED}` | | | -| [`FUZZER`](http://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake) | `` | Enable fuzzer: libfuzzer | | -| [`PARALLEL_COMPILE_JOBS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake) | `` | Define the maximum number of concurrent compilation jobs" " | | -| [`PARALLEL_LINK_JOBS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake) | `` | Define the maximum number of concurrent link jobs" " | | -| [`SANITIZE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake) | `` | Enable sanitizer: address, memory, thread, undefined" " | | -| [`LINKER_NAME`](http://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake) | `` | Linker name or full path | | -| [`WEVERYTHING`](http://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | -| [`ENABLE_AMQPCPP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake) | `${ENABLE_LIBRARIES}` | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake) | `${ENABLE_LIBRARIES}` | Enable Avro | | -| [`ENABLE_BASE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake) | `64 "Enable base64" ${ENABLE_LIBRARIES}` | | | -| [`ENABLE_BROTLI`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `${ENABLE_LIBRARIES}` | Enable brotli | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | -| [`ENABLE_CAPNP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake) | `${ENABLE_LIBRARIES}` | Enable Cap'n Proto | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system capnproto library instead of bundled | | -| [`ENABLE_CASSANDRA`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake) | `${ENABLE_LIBRARIES}` | Enable Cassandra | | -| [`ENABLE_CCACHE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake) | `${ENABLE_CCACHE_BY_DEFAULT}` | Speedup re-compilations using ccache | | -| [`ENABLE_CURL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake) | `${ENABLE_LIBRARIES}` | Enable curl | | -| [`USE_INTERNAL_CURL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake) | `${NOT_UNBUNDLED}` | Use internal curl library | | -| [`USE_LIBCXX`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake) | `${NOT_UNBUNDLED}` | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake) | `${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}` | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | -| [`ENABLE_FASTOPS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake) | `${ENABLE_LIBRARIES}` | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_GPERF`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake) | `${ENABLE_LIBRARIES}` | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake) | `${ENABLE_LIBRARIES}` | Use gRPC | | -| [`ENABLE_GTEST_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake) | `${ENABLE_LIBRARIES}` | Enable gtest library | | -| [`USE_INTERNAL_GTEST_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system Google Test instead of bundled | | -| [`ENABLE_H`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake) | `3 "Enable H3" ${ENABLE_LIBRARIES}` | | | -| [`ENABLE_HDFS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake) | `${ENABLE_LIBRARIES}` | Enable HDFS | | -| [`USE_INTERNAL_HDFS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake) | `3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk` | | | -| [`ENABLE_ICU`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `${ENABLE_LIBRARIES}` | Enable ICU | | -| [`ENABLE_ICU`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `0` | Enable ICU | | -| [`USE_INTERNAL_ICU_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system ICU library instead of bundled | | -| [`ENABLE_LDAP`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake) | `${ENABLE_LIBRARIES}` | Enable LDAP | | -| [`USE_INTERNAL_LDAP_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system *LDAP library instead of bundled | | -| [`ENABLE_GSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `${ENABLE_LIBRARIES}` | Enable gsasl library | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `${USE_STATIC_LIBRARIES}` | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake) | `2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ${NOT_UNBUNDLED}` | | | -| [`ENABLE_EMBEDDED_COMPILER`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `${ENABLE_LIBRARIES}` | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`USE_INTERNAL_LLVM_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `${NOT_UNBUNDLED}` | Use bundled or system LLVM library. | | -| [`LLVM_HAS_RTTI`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`ENABLE_MSGPACK`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake) | `${ENABLE_LIBRARIES}` | Enable msgpack library | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system msgpack library instead of bundled | | -| [`ENABLE_MYSQL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `${ENABLE_LIBRARIES}` | Enable MySQL | | -| [`ENABLE_MYSQL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `FALSE` | Enable MySQL | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system mysqlclient library instead of bundled | | -| [`ENABLE_ODBC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake) | `${ENABLE_LIBRARIES}` | Enable ODBC library | | -| [`USE_INTERNAL_ODBC_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake) | `${NOT_UNBUNDLED}` | Use internal ODBC library | | -| [`ENABLE_OPENCL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake) | `${ENABLE_LIBRARIES}` | Enable OpenCL support | | -| [`ENABLE_ORC`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake) | `${ENABLE_LIBRARIES}` | Enable ORC | | -| [`USE_INTERNAL_ORC_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake) | `"Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk` | | | -| [`ENABLE_PARQUET`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake) | `${ENABLE_LIBRARIES}` | Enable parquet | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake) | `ON` | Use internal Poco library | | -| [`ENABLE_PROTOBUF`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake) | `${ENABLE_LIBRARIES}` | Enable protobuf | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system protobuf instead of bundled | | -| [`ENABLE_RAPIDJSON`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake) | `${ENABLE_LIBRARIES}` | Use rapidjson | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system rapidjson library instead of bundled | | -| [`ENABLE_RDKAFKA`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake) | `${ENABLE_LIBRARIES}` | Enable kafka | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake) | `2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ${NOT_UNBUNDLED}` | | | -| [`ENABLE_S`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake) | `3 "Enable S3" ${ENABLE_LIBRARIES}` | | | -| [`USE_INTERNAL_AWS_S`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake) | `3_LIBRARY "Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk` | | | -| [`USE_SENTRY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake) | `${ENABLE_LIBRARIES}` | Use Sentry | | -| [`USE_SIMDJSON`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake) | `${ENABLE_LIBRARIES}` | Use simdjson | | -| [`USE_SNAPPY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake) | `${ENABLE_LIBRARIES}` | Enable support of snappy library | | -| [`USE_INTERNAL_SNAPPY_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system snappy library instead of bundled | | -| [`ENABLE_SSL`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake) | `${ENABLE_LIBRARIES}` | Enable ssl | | -| [`USE_INTERNAL_SSL_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system *ssl library instead of bundled | | -| [`ENABLE_STATS`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake) | `${ENABLE_LIBRARIES}` | Enalbe StatsLib library | | -| [`USE_UNWIND`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake) | `${ENABLE_LIBRARIES}` | Enable libunwind (better stacktraces) | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](http://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake) | `${NOT_UNBUNDLED}` | Set to FALSE to use system zstd library instead of bundled | | -| [`ENABLE_CLICKHOUSE_ALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `ON` | Enable all tools | | -| [`ENABLE_CLICKHOUSE_SERVER`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-server | | -| [`ENABLE_CLICKHOUSE_CLIENT`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-client | | -| [`ENABLE_CLICKHOUSE_LOCAL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-local | | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-benchmark | | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-extract-from-config | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-compressor | | -| [`ENABLE_CLICKHOUSE_COPIER`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-copier | | -| [`ENABLE_CLICKHOUSE_FORMAT`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-format | | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-obfuscator | | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-odbc-bridge | | -| [`ENABLE_CLICKHOUSE_INSTALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `OFF` | Enable clickhouse-install | | -| [`ENABLE_CLICKHOUSE_INSTALL`](http://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt) | `${ENABLE_CLICKHOUSE_ALL}` | Enable clickhouse-install | | -| [`ENABLE_MULTITARGET_CODE`](http://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt) | `ON` | | | diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py new file mode 100755 index 00000000000..7e919934ab1 --- /dev/null +++ b/cmake_flags_generator.py @@ -0,0 +1,104 @@ +import re +import os.path +from typing import TextIO, List, Tuple, Optional + +Entity = Tuple[str, str, str] + +# https://regex101.com/r/R6iogw/11 +cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" + +output_file_name: str = "cmake_flags_and_output.md" +header_file_name: str = "cmake_files_header.md" +footer_file_name: str = "cmake_files_footer.md" + +ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" + +name_str: str = "(`{name}`)[" + ch_master_url + "{path}#L{line}]" +default_anchor_str: str = "(`{name}`)[#{anchor}]" + +def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> str: + (line, comment) = line_comment + (_name, description, default) = entity + + def anchor(t: str) -> str: + return ''.join([i.lower() for i in t if i.isalpha() or i == "_"]) + + if (len(default) == 0): + default = "`OFF`" + elif default[0] == "$": + default = default[2:-1] + default = default_anchor_str.format( + name=default, + anchor=anchor(default)) + else: + default = "`" + default + "`" + + name: str = name_str.format( + anchor=anchor(_name), + name=_name, + path=path, + line=line) + + return "| " + name + " | " + default + " | " + description + " | " + comment + " |" + +def process_file(input_name: str) -> List[str]: + print("Processing", input_name) + + out: List[str] = [] + + with open(input_name, 'r') as cmake_file: + contents: str = cmake_file.read() + + def get_line_and_comment(target: str) -> Tuple[int, str]: + contents_list: List[str] = contents.split("\n") + comment: str = "" + + for n, line in enumerate(contents_list): + if line.find(target) == -1: + continue + + for maybe_comment_line in contents_list[n::-1]: + if (re.match("\s*#\s*", maybe_comment_line)): + comment = re.sub("\s*#\s*", "", maybe_comment_line) + "\n" + comment + else: + break + + return n, comment + + matches: Optional[List[Entity]] = re.findall(cmake_option_regex, contents, re.MULTILINE) + + if matches: + for entity in matches: + out.append(build_entity(input_name, entity, get_line_and_comment(entity[0]))) + + return out + +def write_file(output: TextIO, in_file_name: str) -> None: + output.write("\n".join(sorted(process_file(in_file_name)))) + +def process_folder(output: TextIO, name: str) -> None: + for root, _, files in os.walk(name): + print("Processing ", root) + + for f in files: + if f == "CMakeLists.txt" or ".cmake" in f: + write_file(output, root + "/" + f) + +def process() -> None: + with open(output_file_name, "w") as f: + with open(header_file_name, "r") as header: + f.write(header.read()) + + write_file(f, "CMakeLists.txt") + write_file(f, "PreLoad.cmake") + + process_folder(f, "base") + process_folder(f, "cmake") + process_folder(f, "src") + process_folder(f, "programs") + process_folder(f, "utils") + + with open(footer_file_name, "r") as footer: + f.write(footer.read()) + +process() diff --git a/cmake_flags_md_generator.sh b/cmake_flags_md_generator.sh deleted file mode 100755 index 17ccc8a1bba..00000000000 --- a/cmake_flags_md_generator.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash/ - -# https://regex101.com/r/R6iogw/7 - -output_file_name="cmake_flags_and_output.md" -ch_master_url="http:\/\/github.com\/clickhouse\/clickhouse\/blob\/master\/" - -rm -fr ${output_file_name} -touch ${output_file_name} -cat cmake_files_header.md >> ${output_file_name} - -process() { - for i in "$1"/*.cmake "$1"/CMakeLists.txt;do - echo "Processing $i" - subd_name=${i//\//\\/} - subd_name=${subd_name//\./\\\.} - subd_name=${subd_name:2} - regex='s/^((\s*#\s+.*\n?)*)\s*option\s*\(([A-Z_]+)\s*(\"((.|\n)*?)\")?\s*(.*)?\).*$/| [`\3`]('$ch_master_url${subd_name:2}') | `\7` | \5 | \1 |/mg;t;d' - - if [ -f $i ]; then - cat $i | sed -E "$regex" >> ${output_file_name} - fi - done - - if [ "$2" = true ] ; then - for d in "$1"/*;do - if [ -d "$d" ];then - process $d - fi - done - fi -} - -process ./ false - -for base_folder in ./base ./cmake ./programs ./src; do - process $base_folder true -done From dc94a754ddcf8feb436e249f28092146acc6a6d7 Mon Sep 17 00:00:00 2001 From: myrrc Date: Tue, 15 Sep 2020 23:17:24 +0300 Subject: [PATCH 080/390] minor bugs fixed, slightly modified main cmake --- CMakeLists.txt | 79 ++++++++++++++++------- cmake_flags_and_output.md | 131 ++++++++++++++++++-------------------- cmake_flags_generator.py | 22 +++---- 3 files changed, 127 insertions(+), 105 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 09a4500e314..01fb16f46b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,8 @@ endforeach() project(ClickHouse) option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION - "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) + "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) + but is not possible to satisfy" ON) if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) @@ -60,7 +61,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # Need cmake 3.9+ # Usually impractical. # See also ENABLE_THINLTO -option(ENABLE_IPO "Enable full link time optimization") +option(ENABLE_IPO "Full link time optimization") if(ENABLE_IPO) cmake_policy(SET CMP0069 NEW) @@ -91,12 +92,16 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) -option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) -option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) +option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON) +option(MAKE_STATIC_LIBRARIES "Disable to make shared libraries" ${USE_STATIC_LIBRARIES}) if (NOT MAKE_STATIC_LIBRARIES) - option (SPLIT_SHARED_LIBRARIES "DEV ONLY. Keep all internal libs as separate .so for faster linking" OFF) - option (CLICKHOUSE_SPLIT_BINARY "Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... )" OFF) + # DEVELOPER ONLY. + # Faster linking if turned on. + option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files") + + option(CLICKHOUSE_SPLIT_BINARY + "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled") endif () if (MAKE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES) @@ -111,7 +116,9 @@ if (USE_STATIC_LIBRARIES) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) endif () -option (ENABLE_FUZZING "Enables fuzzing instrumentation" OFF) +# Enable fuzzy testing using libfuzzer +# Implies ${WITH_COVERAGE} +option (ENABLE_FUZZING "Enables fuzzing instrumentation") if (ENABLE_FUZZING) message (STATUS "Fuzzing instrumentation enabled") @@ -143,10 +150,13 @@ if (COMPILER_CLANG) endif () endif () -option (ENABLE_TESTS "Enables tests" ON) +# Adds a Google.Test target binary containing unit tests. +option (ENABLE_TESTS ON) if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") - option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY." ON) + # Only for Linux, x86_64. + # Implies ${ENABLE_FASTMEMCPY} + option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON) elseif(GLIBC_COMPATIBILITY) message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration") endif () @@ -179,7 +189,8 @@ else () set(NO_WHOLE_ARCHIVE --no-whole-archive) endif () -option (ADD_GDB_INDEX_FOR_GOLD "Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used." 0) +option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used.") + if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") if (LINKER_NAME STREQUAL "lld") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") @@ -200,6 +211,7 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") endif() cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd + if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000) option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON) endif() @@ -213,7 +225,8 @@ if(NOT DISABLE_CPU_OPTIMIZE) include(cmake/cpu_features.cmake) endif() -option(ARCH_NATIVE "Enable -march=native compiler flag" 0) +option(ARCH_NATIVE "Add -march=native compiler flag") + if (ARCH_NATIVE) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native") endif () @@ -224,6 +237,7 @@ if (UNBUNDLED AND (COMPILER_GCC OR COMPILER_CLANG)) else() set (_CXX_STANDARD "-std=c++2a") endif() + # cmake < 3.12 doesn't support 20. We'll set CMAKE_CXX_FLAGS for now # set (CMAKE_CXX_STANDARD 20) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_CXX_STANDARD}") @@ -236,7 +250,8 @@ if (COMPILER_GCC OR COMPILER_CLANG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") endif () -option(WITH_COVERAGE "Build with coverage." 0) +# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc +option(WITH_COVERAGE) if (WITH_COVERAGE AND COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") @@ -270,10 +285,13 @@ if (COMPILER_CLANG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") if (NOT ENABLE_TESTS AND NOT SANITIZE) - option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) + # https://clang.llvm.org/docs/ThinLTO.html + # Applies to clang only. + # Disabled when building with tests or sanitizers. + option(ENABLE_THINLTO ON) endif() - # We cannot afford to use LTO when compiling unitests, and it's not enough + # We cannot afford to use LTO when compiling unit tests, and it's not enough # to only supply -fno-lto at the final linking stage. So we disable it # completely. if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) @@ -286,8 +304,8 @@ if (COMPILER_CLANG) endif () # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled - find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") + if (LLVM_AR_PATH) message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.") set (CMAKE_AR ${LLVM_AR_PATH}) @@ -306,20 +324,25 @@ elseif (ENABLE_THINLTO) message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang") endif () -option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) +option(ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) + +# We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your +# system. +# This mode exists for enthusiastic developers who are searching for trouble. +# Useful for maintainers of OS packages. +option (UNBUNDLED "Use system libraries instead of ones in contrib/") -option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) - set(NOT_UNBUNDLED 0) + set(NOT_UNBUNDLED OFF) else () - set(NOT_UNBUNDLED 1) + set(NOT_UNBUNDLED ON) endif () if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN)) # Using system libs can cause a lot of warnings in includes (on macro expansion). - option (WERROR "Enable -Werror compiler option" OFF) + option(WERROR "Enable -Werror compiler option") else () - option (WERROR "Enable -Werror compiler option" ON) + option(WERROR "Enable -Werror compiler option" ON) endif () if (WERROR) @@ -361,8 +384,9 @@ else () set (CMAKE_POSITION_INDEPENDENT_CODE ON) endif () -# Using "include-what-you-use" tool. -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) +# https://github.com/include-what-you-use/include-what-you-use +option (USE_INCLUDE_WHAT_YOU_USE) + if (USE_INCLUDE_WHAT_YOU_USE) find_program(IWYU_PATH NAMES include-what-you-use iwyu) if (NOT IWYU_PATH) @@ -376,6 +400,7 @@ endif () if (ENABLE_TESTS) message (STATUS "Tests are enabled") endif () + enable_testing() # Enable for tests without binary # when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc @@ -385,7 +410,13 @@ else () set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc") endif () -message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") +message (STATUS + "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; + USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} + MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} + SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} + UNBUNDLED=${UNBUNDLED} + CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") include (GNUInstallDirs) include (cmake/contrib_finder.cmake) diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 11f5ed36496..42d6248e94f 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -12,75 +12,68 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| (`ADD_GDB_INDEX_FOR_GOLD`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L181] | `0` | Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| (`ARCH_NATIVE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L215] | `0` | Enable -march=native compiler flag | | -| (`CLICKHOUSE_SPLIT_BINARY`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L98] | `OFF` | Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... ) | | -| (`COMPILER_PIPE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L203] | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | -| (`ENABLE_FUZZING`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L113] | `OFF` | Enables fuzzing instrumentation | | -| (`ENABLE_IPO`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62] | `OFF` | Enable full link time optimization | | -| (`ENABLE_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L308] | `ON` | Enable all libraries (Global default switch) | | -| (`ENABLE_TESTS`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L145] | `ON` | Enables tests | | -| (`ENABLE_THINLTO`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L61] | `ON` | Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers. | Need cmake 3.9+ -Usually impractical. -See also ENABLE_THINLTO - | -| (`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30] | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | -| (`GLIBC_COMPATIBILITY`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L148] | `ON` | Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY. | | -| (`MAKE_STATIC_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to make shared libraries | | -| (`SPLIT_SHARED_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L97] | `OFF` | DEV ONLY. Keep all internal libs as separate .so for faster linking | | -| (`UNBUNDLED`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L147] | `OFF` | Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages. | | -| (`USE_INCLUDE_WHAT_YOU_USE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L364] | `OFF` | Use 'include-what-you-use' tool | | -| (`USE_STATIC_LIBRARIES`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L93] | `ON` | Set to FALSE to use shared libraries | | -| (`WERROR`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L319] | `OFF` | Enable -Werror compiler option | | -| (`WERROR`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L319] | `ON` | Enable -Werror compiler option | | -| (`WITH_COVERAGE`)[https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L117] | `0` | Build with coverage. | || (`WEVERYTHING`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20] | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | || (`PARALLEL_COMPILE_JOBS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8] | `""` | Define the maximum number of concurrent compilation jobs | | -| (`PARALLEL_LINK_JOBS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21] | `""` | Define the maximum number of concurrent link jobs | || (`ENABLE_CLANG_TIDY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2] | `OFF` | Use 'clang-tidy' static analyzer if present | || (`SANITIZE`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0] | `""` | Enable sanitizer: address, memory, thread, undefined | || (`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use bundled library ${LIB_NAME} instead of system | || (`FUZZER`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0] | `OFF` | Enable fuzzer: libfuzzer | || (`LINKER_NAME`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42] | `OFF` | Linker name or full path | || (`ENABLE_SSL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ssl | | -| (`USE_INTERNAL_SSL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system *ssl library instead of bundled | || (`ENABLE_GPERF`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use gperf function hash generator tool | || (`USE_INTERNAL_SPARSEHASH_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0] | `ON` | Set to FALSE to use system sparsehash library instead of bundled | || (`ENABLE_BASE64`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable base64 | || (`USE_INTERNAL_RE2_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system re2 library instead of bundled [slower] | || (`ENABLE_H3`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable H3 | | -| (`USE_INTERNAL_H3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2] | `ON` | Set to FALSE to use system h3 library instead of bundled | || (`ENABLE_OPENCL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable OpenCL support | || (`USE_INTERNAL_LIBCXX_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3] | (`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`)[#use_internal_libcxx_library_default] | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | -| (`USE_LIBCXX`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use libc++ and libc++abi instead of libstdc++ | || (`ENABLE_MYSQL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable MySQL | | -| (`ENABLE_MYSQL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1] | `FALSE` | Enable MySQL | | -| (`USE_INTERNAL_MYSQL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system mysqlclient library instead of bundled | || (`USE_INTERNAL_ZLIB_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system zlib library instead of bundled | || (`USE_SIMDJSON`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use simdjson | || (`USE_INTERNAL_SNAPPY_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system snappy library instead of bundled | | -| (`USE_SNAPPY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable support of snappy library | || (`USE_SENTRY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use Sentry | || (`USE_INTERNAL_ZSTD_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system zstd library instead of bundled | || (`ENABLE_LDAP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable LDAP | | -| (`USE_INTERNAL_LDAP_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system *LDAP library instead of bundled | || (`ENABLE_CURL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable curl | | -| (`USE_INTERNAL_CURL`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use internal curl library | || (`ENABLE_ODBC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ODBC library | | -| (`USE_INTERNAL_ODBC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use internal ODBC library | || (`USE_INTERNAL_LIBXML2_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system libxml2 library instead of bundled | || (`ENABLE_STATS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enalbe StatsLib library | || (`ENABLE_AVRO`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Avro | | -| (`USE_INTERNAL_AVRO_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3] | `ON` | Set to FALSE to use system avro library instead of bundled | || (`ENABLE_GTEST_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable gtest library | | -| (`USE_INTERNAL_GTEST_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system Google Test instead of bundled | || (`ENABLE_AMQPCPP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enalbe AMQP-CPP | || (`ENABLE_ICU`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ICU | | -| (`ENABLE_ICU`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1] | `0` | Enable ICU | | -| (`USE_INTERNAL_ICU_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system ICU library instead of bundled | || (`ENABLE_CCACHE`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6] | (`ENABLE_CCACHE_BY_DEFAULT`)[#enable_ccache_by_default] | Speedup re-compilations using ccache | || (`ENABLE_PARQUET`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable parquet | | -| (`USE_INTERNAL_PARQUET_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system parquet library instead of bundled | || (`USE_UNWIND`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable libunwind (better stacktraces) | || (`ENABLE_BROTLI`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable brotli | | -| (`USE_INTERNAL_BROTLI_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to use system libbrotli library instead of bundled | | -| (`USE_INTERNAL_BROTLI_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3] | `ON` | Set to FALSE to use system libbrotli library instead of bundled | || (`ENABLE_MSGPACK`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable msgpack library | | -| (`USE_INTERNAL_MSGPACK_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system msgpack library instead of bundled | || (`ENABLE_EMBEDDED_COMPILER`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| (`LLVM_HAS_RTTI`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39] | `ON` | Enable if LLVM was build with RTTI enabled | | -| (`USE_INTERNAL_LLVM_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7] | (`NOT_UNBUNDLED`)[#not_unbundled] | Use bundled or system LLVM library. | || (`ENABLE_HDFS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable HDFS | | -| (`USE_INTERNAL_HDFS3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2] | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | || (`ENABLE_GSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable gsasl library | | -| (`USE_INTERNAL_LIBGSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3] | (`USE_STATIC_LIBRARIES`)[#use_static_libraries] | Set to FALSE to use system libgsasl library instead of bundled | | -| (`USE_INTERNAL_LIBGSASL_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3] | `ON` | Set to FALSE to use system libgsasl library instead of bundled | || (`ENABLE_FASTOPS`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable fast vectorized mathematical functions library by Mikhail Parakhin | || (`ENABLE_RDKAFKA`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable kafka | | -| (`USE_INTERNAL_RDKAFKA_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system librdkafka instead of the bundled | || (`ENABLE_ORC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable ORC | | -| (`USE_INTERNAL_ORC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3] | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | || (`ENABLE_CASSANDRA`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Cassandra | || (`ENABLE_S3`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable S3 | | -| (`USE_INTERNAL_AWS_S3_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2] | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | || (`ENABLE_RAPIDJSON`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use rapidjson | | -| (`USE_INTERNAL_RAPIDJSON_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system rapidjson library instead of bundled | || (`USE_INTERNAL_POCO_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0] | `ON` | Use internal Poco library | || (`ENABLE_GRPC`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Use gRPC | | -| (`USE_INTERNAL_GRPC_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | || (`ENABLE_PROTOBUF`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable protobuf | | -| (`USE_INTERNAL_PROTOBUF_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system protobuf instead of bundled | || (`ENABLE_CAPNP`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0] | (`ENABLE_LIBRARIES`)[#enable_libraries] | Enable Cap'n Proto | | -| (`USE_INTERNAL_CAPNP_LIBRARY`)[https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3] | (`NOT_UNBUNDLED`)[#not_unbundled] | Set to FALSE to use system capnproto library instead of bundled | || (`ENABLE_MULTITARGET_CODE`)[https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L117] | `ON` | | | -| (`STRIP_DEBUG_SYMBOLS_FUNCTIONS`)[https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64] | (`STRIP_DSF_DEFAULT`)[#strip_dsf_default] | Do not generate debugger info for ClickHouse functions. - Provides faster linking and lower binary size. - Tradeoff is the inability to debug some source files with e.g. gdb - (empty stack frames and no local variables). | || (`ENABLE_CLICKHOUSE_ALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8] | `ON` | Enable all tools | | -| (`ENABLE_CLICKHOUSE_BENCHMARK`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-benchmark | | -| (`ENABLE_CLICKHOUSE_CLIENT`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-client | | -| (`ENABLE_CLICKHOUSE_COMPRESSOR`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-compressor | | -| (`ENABLE_CLICKHOUSE_COPIER`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-copier | | -| (`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-extract-from-config | | -| (`ENABLE_CLICKHOUSE_FORMAT`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-format | | -| (`ENABLE_CLICKHOUSE_INSTALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-install | | -| (`ENABLE_CLICKHOUSE_INSTALL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21] | `OFF` | Enable clickhouse-install | | -| (`ENABLE_CLICKHOUSE_LOCAL`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L11] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-local | | -| (`ENABLE_CLICKHOUSE_OBFUSCATOR`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-obfuscator | | -| (`ENABLE_CLICKHOUSE_ODBC_BRIDGE`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-odbc-bridge | | -| (`ENABLE_CLICKHOUSE_SERVER`)[https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9] | (`ENABLE_CLICKHOUSE_ALL`)[#enable_clickhouse_all] | Enable clickhouse-server | | -## Developer's guide for adding new CMake options +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L191) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | +| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L102) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L215) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Enables fuzzing instrumentation | Enable fuzzy testing using libfuzzer. Implies ${WITH_COVERAGE}. | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L326) | `ON` | Enable all libraries (Global default switch) | | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | | Adds a Google.Test target binary containing unit tests.. | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62) | `ON` | | Need cmake 3.9+. Usually impractical.. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | +| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L100) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L387) | `OFF` | | https://github.com/include-what-you-use/include-what-you-use. | +| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94) | `ON` | Disable to use shared libraries | | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | | Enable fuzzy testing using libfuzzer. || [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | || [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8) | `""` | Define the maximum number of concurrent compilation jobs | | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21) | `""` | Define the maximum number of concurrent link jobs | || [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2) | `OFF` | Use 'clang-tidy' static analyzer if present | || [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0) | `""` | Enable sanitizer: address, memory, thread, undefined | || [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | || [`FUZZER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0) | `OFF` | Enable fuzzer: libfuzzer | || [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42) | `OFF` | Linker name or full path | || [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | || [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | || [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | || [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | || [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | || [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | || [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | || [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | || [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | || [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | || [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | || [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | || [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | || [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | || [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | +| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | || [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | || [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | || [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | || [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | || [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | || [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | +| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | || [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | || [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | +| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | || [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | || [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | || [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | || [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | || [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | || [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | +| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | || [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | || [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | || [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | || [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | || [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | || [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | || [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | || [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | || [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0) | `ON` | Use internal Poco library | || [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | || [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | || [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | || [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L117) | `ON` | | | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions. Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables). | || [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | `ON` | Enable all tools | | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-benchmark | | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-client | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-compressor | | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-copier | | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-extract-from-config | | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-format | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-install | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | `OFF` | Enable clickhouse-install | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L11) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-local | | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-obfuscator | | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-odbc-bridge | | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-server | |## Developer's guide for adding new CMake options ### Don't be obvious. Be informative. diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 7e919934ab1..9b77c91f395 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -13,15 +13,15 @@ footer_file_name: str = "cmake_files_footer.md" ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" -name_str: str = "(`{name}`)[" + ch_master_url + "{path}#L{line}]" -default_anchor_str: str = "(`{name}`)[#{anchor}]" +name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" +default_anchor_str: str = "[`{name}`](#{anchor})" def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> str: (line, comment) = line_comment - (_name, description, default) = entity + (_name, _description, default) = entity def anchor(t: str) -> str: - return ''.join([i.lower() for i in t if i.isalpha() or i == "_"]) + return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) if (len(default) == 0): default = "`OFF`" @@ -39,11 +39,11 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> st path=path, line=line) + description: str = "".join(_description.split("\n")) + return "| " + name + " | " + default + " | " + description + " | " + comment + " |" def process_file(input_name: str) -> List[str]: - print("Processing", input_name) - out: List[str] = [] with open(input_name, 'r') as cmake_file: @@ -57,12 +57,12 @@ def process_file(input_name: str) -> List[str]: if line.find(target) == -1: continue - for maybe_comment_line in contents_list[n::-1]: - if (re.match("\s*#\s*", maybe_comment_line)): - comment = re.sub("\s*#\s*", "", maybe_comment_line) + "\n" + comment - else: + for maybe_comment_line in contents_list[n - 1::-1]: + if not re.match("\s*#\s*", maybe_comment_line): break + comment = re.sub("\s*#\s*", "", maybe_comment_line) + ". " + comment + return n, comment matches: Optional[List[Entity]] = re.findall(cmake_option_regex, contents, re.MULTILINE) @@ -78,8 +78,6 @@ def write_file(output: TextIO, in_file_name: str) -> None: def process_folder(output: TextIO, name: str) -> None: for root, _, files in os.walk(name): - print("Processing ", root) - for f in files: if f == "CMakeLists.txt" or ".cmake" in f: write_file(output, root + "/" + f) From 29ac26f0f2ac30b7515d3fde0f57131ca14b1384 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 16 Sep 2020 00:12:37 +0300 Subject: [PATCH 081/390] separated ch parts table --- cmake_files_footer.md | 1 + cmake_flags_and_output.md | 32 +++++++++++++---------- cmake_flags_generator.py | 43 +++++++++++++++++++++---------- programs/CMakeLists.txt | 49 +++++++++++++++++++++++------------- src/Functions/CMakeLists.txt | 10 +++----- 5 files changed, 86 insertions(+), 49 deletions(-) diff --git a/cmake_files_footer.md b/cmake_files_footer.md index b67f79b9b65..a35d39cf9d3 100644 --- a/cmake_files_footer.md +++ b/cmake_files_footer.md @@ -1,3 +1,4 @@ + ## Developer's guide for adding new CMake options ### Don't be obvious. Be informative. diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 42d6248e94f..c0b9494dfcf 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -61,19 +61,25 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | || [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | | [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | || [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | | [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | || [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L117) | `ON` | | | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions. Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables). | || [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | `ON` | Enable all tools | | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-benchmark | | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-client | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-compressor | | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-copier | | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-extract-from-config | | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-format | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-install | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | `OFF` | Enable clickhouse-install | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L11) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-local | | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-obfuscator | | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-odbc-bridge | | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Enable clickhouse-server | |## Developer's guide for adding new CMake options +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions. Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables). | | + +### ClickHouse additory parts +| Name | Default value | Description | +|------|---------------|-------------| +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | `OFF` | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L33) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | +## Developer's guide for adding new CMake options ### Don't be obvious. Be informative. diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 9b77c91f395..2d169318418 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -16,34 +16,37 @@ ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" default_anchor_str: str = "[`{name}`](#{anchor})" -def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> str: +def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **options) -> str: (line, comment) = line_comment (_name, _description, default) = entity - def anchor(t: str) -> str: + def make_anchor(t: str) -> str: return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) - if (len(default) == 0): + if len(default) == 0: default = "`OFF`" elif default[0] == "$": default = default[2:-1] default = default_anchor_str.format( name=default, - anchor=anchor(default)) + anchor=make_anchor(default)) else: default = "`" + default + "`" name: str = name_str.format( - anchor=anchor(_name), + anchor=make_anchor(_name), name=_name, path=path, line=line) - description: str = "".join(_description.split("\n")) + if options.get("no_desc", False): + description: str = "" + else: + description: str = "".join(_description.split("\n")) + " | " - return "| " + name + " | " + default + " | " + description + " | " + comment + " |" + return "| " + name + " | " + default + " | " + description + comment + " |" -def process_file(input_name: str) -> List[str]: +def process_file(input_name: str, **options) -> List[str]: out: List[str] = [] with open(input_name, 'r') as cmake_file: @@ -69,12 +72,17 @@ def process_file(input_name: str) -> List[str]: if matches: for entity in matches: - out.append(build_entity(input_name, entity, get_line_and_comment(entity[0]))) + out.append( + build_entity( + input_name, + entity, + get_line_and_comment(entity[0]), + **options)) return out -def write_file(output: TextIO, in_file_name: str) -> None: - output.write("\n".join(sorted(process_file(in_file_name)))) +def write_file(output: TextIO, in_file_name: str, **options) -> None: + output.write("\n".join(sorted(process_file(in_file_name, **options)))) def process_folder(output: TextIO, name: str) -> None: for root, _, files in os.walk(name): @@ -93,8 +101,17 @@ def process() -> None: process_folder(f, "base") process_folder(f, "cmake") process_folder(f, "src") - process_folder(f, "programs") - process_folder(f, "utils") + + # Various ClickHouse extern parts (Copier/Obfuscator/...) + + f.write(""" + +### ClickHouse additory parts +| Name | Default value | Description | +|------|---------------|-------------| +""") + + write_file(f, "programs/CMakeLists.txt", no_desc=True) with open(footer_file_name, "r") as footer: f.write(footer.read()) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 89220251cda..5d07f0bf6e1 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,30 +2,45 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -# 'clickhouse' binary is a multi purpose tool, -# that contain multiple execution modes (client, server, etc.) -# each of them is built and linked as a separate library, defined below. +# The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), +# each of them may be built and linked as a separate library. +option (ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse tools by default" ON) -option (ENABLE_CLICKHOUSE_ALL "Enable all tools" ON) -option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_SERVER ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_CLIENT ${ENABLE_CLICKHOUSE_ALL}) + +# https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ +option (ENABLE_CLICKHOUSE_LOCAL ${ENABLE_CLICKHOUSE_ALL}) + +# https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ +option (ENABLE_CLICKHOUSE_BENCHMARK ${ENABLE_CLICKHOUSE_ALL}) + +# ??? +option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG ${ENABLE_CLICKHOUSE_ALL}) + +# ??? +option (ENABLE_CLICKHOUSE_COMPRESSOR ${ENABLE_CLICKHOUSE_ALL}) + +# https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ +option (ENABLE_CLICKHOUSE_COPIER ${ENABLE_CLICKHOUSE_ALL}) + +# ??? +option (ENABLE_CLICKHOUSE_FORMAT ${ENABLE_CLICKHOUSE_ALL}) + +# ??? +option (ENABLE_CLICKHOUSE_OBFUSCATOR ${ENABLE_CLICKHOUSE_ALL}) + +# ??? +option (ENABLE_CLICKHOUSE_ODBC_BRIDGE ${ENABLE_CLICKHOUSE_ALL}) if (CLICKHOUSE_SPLIT_BINARY) - option (ENABLE_CLICKHOUSE_INSTALL "Enable clickhouse-install" OFF) + option(ENABLE_CLICKHOUSE_INSTALL) else () - option (ENABLE_CLICKHOUSE_INSTALL "Enable clickhouse-install" ${ENABLE_CLICKHOUSE_ALL}) + option(ENABLE_CLICKHOUSE_INSTALL ${ENABLE_CLICKHOUSE_ALL}) endif () if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES)) - set(CLICKHOUSE_ONE_SHARED 1) + set(CLICKHOUSE_ONE_SHARED ON) endif() configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 0a99a034a33..fd37e14685f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -62,12 +62,10 @@ else() endif() -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - Provides faster linking and lower binary size. - Tradeoff is the inability to debug some source files with e.g. gdb - (empty stack frames and no local variables)." - ${STRIP_DSF_DEFAULT}) +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions" ${STRIP_DSF_DEFAULT}) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) message(WARNING "Not generating debugger info for ClickHouse functions") From 42b22c34247bfd85316855da5d217518b2333c8b Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 16 Sep 2020 00:35:06 +0300 Subject: [PATCH 082/390] fixed handling of conditional options --- CMakeLists.txt | 2 +- cmake_flags_and_output.md | 138 ++++++++++++++++++++++++-------------- cmake_flags_generator.py | 63 +++++++---------- 3 files changed, 115 insertions(+), 88 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01fb16f46b8..eda020be25e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -385,7 +385,7 @@ else () endif () # https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE) +option (USE_INCLUDE_WHAT_YOU_USE OFF) if (USE_INCLUDE_WHAT_YOU_USE) find_program(IWYU_PATH NAMES include-what-you-use iwyu) diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index c0b9494dfcf..55f9df7887e 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -16,69 +16,109 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | | [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L102) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | | [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L215) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2) | `OFF` | Use 'clang-tidy' static analyzer if present | | +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | Enable all ClickHouse tools by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L33) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | | [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Enables fuzzing instrumentation | Enable fuzzy testing using libfuzzer. Implies ${WITH_COVERAGE}. | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | | [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | +| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | | [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L326) | `ON` | Enable all libraries (Global default switch) | | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | | [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | | Adds a Google.Test target binary containing unit tests.. | | [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62) | `ON` | | Need cmake 3.9+. Usually impractical.. | | [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | +| [`FUZZER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0) | `OFF` | Enable fuzzer: libfuzzer | | | [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42) | `OFF` | Linker name or full path | | +| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | | [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8) | `""` | Define the maximum number of concurrent compilation jobs | | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21) | `""` | Define the maximum number of concurrent link jobs | | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0) | `""` | Enable sanitizer: address, memory, thread, undefined | | | [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L100) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size.. Tradeoff is the inability to debug some source files with e.g. gdb. (empty stack frames and no local variables).". | | [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | | [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L387) | `OFF` | | https://github.com/include-what-you-use/include-what-you-use. | +| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | +| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | +| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | +| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | +| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | +| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | +| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | +| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | +| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | +| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | +| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | +| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | +| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | +| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0) | `ON` | Use internal Poco library | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | +| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | +| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | +| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | +| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | | [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94) | `ON` | Disable to use shared libraries | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | | [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | | Enable fuzzy testing using libfuzzer. || [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | || [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8) | `""` | Define the maximum number of concurrent compilation jobs | | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21) | `""` | Define the maximum number of concurrent link jobs | || [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2) | `OFF` | Use 'clang-tidy' static analyzer if present | || [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0) | `""` | Enable sanitizer: address, memory, thread, undefined | || [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | || [`FUZZER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0) | `OFF` | Enable fuzzer: libfuzzer | || [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42) | `OFF` | Linker name or full path | || [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | || [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | || [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | || [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | || [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | || [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | || [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | || [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | || [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | || [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | || [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | || [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | || [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | || [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | || [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | || [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | || [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | || [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | || [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | || [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | || [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | -| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | || [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | || [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | -| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | || [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | || [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | || [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | || [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | || [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | || [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | || [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | || [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | || [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | || [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | || [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | || [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | || [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | || [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | || [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0) | `ON` | Use internal Poco library | || [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | || [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | || [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | || [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L117) | `ON` | | | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L64) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions. Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables). | | +| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | | Enable fuzzy testing using libfuzzer. | -### ClickHouse additory parts -| Name | Default value | Description | -|------|---------------|-------------| -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | `OFF` | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L33) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | ???. | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ## Developer's guide for adding new CMake options ### Don't be obvious. Be informative. diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 2d169318418..79da97f218b 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -1,6 +1,6 @@ import re import os.path -from typing import TextIO, List, Tuple, Optional +from typing import TextIO, List, Tuple, Optional, Dict Entity = Tuple[str, str, str] @@ -16,12 +16,19 @@ ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" default_anchor_str: str = "[`{name}`](#{anchor})" -def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **options) -> str: +# Needed to detect conditional variables (those which are defined twice) +entities: Dict[str, str] = {} + + +def make_anchor(t: str) -> str: + return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) + +def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **options) -> None: (line, comment) = line_comment (_name, _description, default) = entity - def make_anchor(t: str) -> str: - return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) + if _name in entities: + return if len(default) == 0: default = "`OFF`" @@ -44,11 +51,9 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **opt else: description: str = "".join(_description.split("\n")) + " | " - return "| " + name + " | " + default + " | " + description + comment + " |" - -def process_file(input_name: str, **options) -> List[str]: - out: List[str] = [] + entities[name] = "| " + name + " | " + default + " | " + description + comment + " |" +def process_file(input_name: str, **options) -> None: with open(input_name, 'r') as cmake_file: contents: str = cmake_file.read() @@ -72,46 +77,28 @@ def process_file(input_name: str, **options) -> List[str]: if matches: for entity in matches: - out.append( - build_entity( - input_name, - entity, - get_line_and_comment(entity[0]), - **options)) + build_entity(input_name, entity, get_line_and_comment(entity[0])) - return out - -def write_file(output: TextIO, in_file_name: str, **options) -> None: - output.write("\n".join(sorted(process_file(in_file_name, **options)))) - -def process_folder(output: TextIO, name: str) -> None: +def process_folder(name: str) -> None: for root, _, files in os.walk(name): for f in files: if f == "CMakeLists.txt" or ".cmake" in f: - write_file(output, root + "/" + f) + process_file(root + "/" + f) def process() -> None: + process_file("CMakeLists.txt") + process_file("programs/CMakeLists.txt", no_desc=True) + + process_folder("base") + process_folder("cmake") + process_folder("src") + with open(output_file_name, "w") as f: with open(header_file_name, "r") as header: f.write(header.read()) - write_file(f, "CMakeLists.txt") - write_file(f, "PreLoad.cmake") - - process_folder(f, "base") - process_folder(f, "cmake") - process_folder(f, "src") - - # Various ClickHouse extern parts (Copier/Obfuscator/...) - - f.write(""" - -### ClickHouse additory parts -| Name | Default value | Description | -|------|---------------|-------------| -""") - - write_file(f, "programs/CMakeLists.txt", no_desc=True) + for k in sorted(entities.keys()): + f.write(entities[k] + "\n") with open(footer_file_name, "r") as footer: f.write(footer.read()) From c5f3f07983c5ea17c3dca7bda0ade2a732305636 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 16 Sep 2020 09:13:28 +0300 Subject: [PATCH 083/390] Translated into Russian. --- docs/ru/commercial/index.md | 10 ++++++++ .../mergetree-family/mergetree.md | 5 +--- docs/ru/operations/system-tables/tables.md | 7 ++++-- docs/ru/sql-reference/statements/index.md | 24 +++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/ru/commercial/index.md b/docs/ru/commercial/index.md index 6bc3c781161..c6c440c17e8 100644 --- a/docs/ru/commercial/index.md +++ b/docs/ru/commercial/index.md @@ -6,4 +6,14 @@ toc_title: "\u041A\u043E\u043C\u043C\u0435\u0440\u0447\u0435\u0441\u043A\u0438\u \ \u0443\u0441\u043B\u0443\u0433\u0438" --- +# Коммерческие услуги {#clickhouse-commercial-services} +Данный раздел содержит описание коммерческих услуг, предоставляемых для ClickHouse. Поставщики этих услуг — независимые компании, которые могут не быть аффилированы с Яндексом. + +Категории услуг: + +- Облачные услуги [Cloud](../commercial/cloud.md) +- Поддержка [Support](../commercial/support.md) + +!!! note "Для поставщиков услуг" + Если вы — представитель компании-поставщика услуг, вы можете отправить запрос на добавление вашей компании и ваших услуг в соответствующий раздел данной документации (или на добавление нового раздела, если ваши услуги не соответствуют ни одной из существующих категорий). Чтобы отправить запрос (pull-request) на добавление описания в документацию, нажмите на значок "карандаша" в правом верхнем углу страницы. Если ваши услуги доступны в только отдельных регионах, не забудьте указать это на соответствующих локализованных страницах (и обязательно отметьте это при отправке заявки). diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 3c80fe663f1..881c11152cd 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -43,9 +43,6 @@ ORDER BY expr Описание параметров смотрите в [описании запроса CREATE](../../../engines/table-engines/mergetree-family/mergetree.md). -!!! note "Примечание" - `INDEX` — экспериментальная возможность, смотрите [Индексы пропуска данных](#table_engine-mergetree-data_skipping-indexes). - ### Секции запроса {#mergetree-query-clauses} - `ENGINE` — имя и параметры движка. `ENGINE = MergeTree()`. `MergeTree` не имеет параметров. @@ -269,7 +266,7 @@ ClickHouse не может использовать индекс, если зн ClickHouse использует эту логику не только для последовательностей дней месяца, но и для любого частично-монотонного первичного ключа. -### Индексы пропуска данных (экспериментальная функциональность) {#table_engine-mergetree-data_skipping-indexes} +### Индексы пропуска данных {#table_engine-mergetree-data_skipping-indexes} Объявление индексов при определении столбцов в запросе `CREATE`. diff --git a/docs/ru/operations/system-tables/tables.md b/docs/ru/operations/system-tables/tables.md index 7b3ea0037b8..52de10871b2 100644 --- a/docs/ru/operations/system-tables/tables.md +++ b/docs/ru/operations/system-tables/tables.md @@ -24,13 +24,16 @@ - [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) - [Distributed](../../engines/table-engines/special/distributed.md#distributed) -- `total_rows` (Nullable(UInt64)) - Общее количество строк, если есть возможность быстро определить точное количество строк в таблице, в противном случае `Null` (включая базовую таблицу `Buffer`). +- `total_rows` (Nullable(UInt64)) - общее количество строк, если есть возможность быстро определить точное количество строк в таблице, в противном случае `Null` (включая базовую таблицу `Buffer`). -- `total_bytes` (Nullable(UInt64)) - Общее количество байт, если можно быстро определить точное количество байт для таблицы на накопителе, в противном случае `Null` (**не включает** в себя никакого базового хранилища). +- `total_bytes` (Nullable(UInt64)) - общее количество байт, если можно быстро определить точное количество байт для таблицы на накопителе, в противном случае `Null` (**не включает** в себя никакого базового хранилища). - Если таблица хранит данные на диске, возвращает используемое пространство на диске (т. е. сжатое). - Если таблица хранит данные в памяти, возвращает приблизительное количество используемых байт в памяти. +- `lifetime_rows` (Nullable(UInt64)) - общее количество строк, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`). + +- `lifetime_bytes` (Nullable(UInt64)) - общее количество байт, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`). Таблица `system.tables` используется при выполнении запроса `SHOW TABLES`. diff --git a/docs/ru/sql-reference/statements/index.md b/docs/ru/sql-reference/statements/index.md index 4dad718d721..c7862015e64 100644 --- a/docs/ru/sql-reference/statements/index.md +++ b/docs/ru/sql-reference/statements/index.md @@ -3,4 +3,28 @@ toc_folder_title: "\u0412\u044B\u0440\u0430\u0436\u0435\u043D\u0438\u044F" toc_priority: 31 --- +# SQL выражения в ClickHouse {#clickhouse-sql-statements} +Выражения описывают различные действия, которые можно выполнить с помощью SQL запросов. Каждый вид выражения имеет свой синтаксис и особенности использования, которые описаны в соответствующих разделах документации: + +- [SELECT](../../sql-reference/statements/select/index.md) +- [INSERT INTO](../../sql-reference/statements/insert-into.md) +- [CREATE](../../sql-reference/statements/create/index.md) +- [ALTER](../../sql-reference/statements/alter/index.md) +- [SYSTEM](../../sql-reference/statements/system.md) +- [SHOW](../../sql-reference/statements/show.md) +- [GRANT](../../sql-reference/statements/grant.md) +- [REVOKE](../../sql-reference/statements/revoke.md) +- [ATTACH](../../sql-reference/statements/attach.md) +- [CHECK TABLE](../../sql-reference/statements/check-table.md) +- [DESCRIBE TABLE](../../sql-reference/statements/describe-table.md) +- [DETACH](../../sql-reference/statements/detach.md) +- [DROP](../../sql-reference/statements/drop.md) +- [EXISTS](../../sql-reference/statements/exists.md) +- [KILL](../../sql-reference/statements/kill.md) +- [OPTIMIZE](../../sql-reference/statements/optimize.md) +- [RENAME](../../sql-reference/statements/rename.md) +- [SET](../../sql-reference/statements/set.md) +- [SET ROLE](../../sql-reference/statements/set-role.md) +- [TRUNCATE](../../sql-reference/statements/truncate.md) +- [USE](../../sql-reference/statements/use.md) From a8e671f0150947f96da602ca90b1f45972f397eb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 11:58:27 +0300 Subject: [PATCH 084/390] Fix MutationsInterpreter. --- src/Interpreters/MutationsInterpreter.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 7f6f3441f5b..30da0d6e65f 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -724,9 +724,7 @@ void MutationsInterpreter::validate() QueryPlan plan; select_interpreter->buildQueryPlan(plan); - addStreamsForLaterStages(stages, plan); - - auto pipeline = plan.buildQueryPipeline(); + auto pipeline = addStreamsForLaterStages(stages, plan); } BlockInputStreamPtr MutationsInterpreter::execute() @@ -737,9 +735,7 @@ BlockInputStreamPtr MutationsInterpreter::execute() QueryPlan plan; select_interpreter->buildQueryPlan(plan); - addStreamsForLaterStages(stages, plan); - - auto pipeline = plan.buildQueryPipeline(); + auto pipeline = addStreamsForLaterStages(stages, plan); BlockInputStreamPtr result_stream = std::make_shared(std::move(*pipeline)); /// Sometimes we update just part of columns (for example UPDATE mutation) From bcea99f2e5104aa6437bf1e2456277c71a99b307 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 16 Sep 2020 11:59:58 +0300 Subject: [PATCH 085/390] Update ThreadPool.cpp --- src/Common/ThreadPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index cb8a7669eef..1255e3d11f8 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -279,7 +279,7 @@ void GlobalThreadPool::initialize(size_t max_threads) { if (the_instance) { - throw Exception(DB::ErrorCodes::LOGICAL_ERROR, + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "The global thread pool is initialized twice"); } From b3d236a535bb3518ad49af36f9b69b79b37ccd08 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 16 Sep 2020 12:42:55 +0300 Subject: [PATCH 086/390] few description fixes --- CMakeLists.txt | 10 ++++------ cmake_files_footer.md | 22 ++++++++++++++-------- cmake_flags_and_output.md | 20 ++++++++++++-------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eda020be25e..e120674347a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,9 +116,8 @@ if (USE_STATIC_LIBRARIES) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) endif () -# Enable fuzzy testing using libfuzzer # Implies ${WITH_COVERAGE} -option (ENABLE_FUZZING "Enables fuzzing instrumentation") +option (ENABLE_FUZZING "Enable fuzzy testing using libfuzzer") if (ENABLE_FUZZING) message (STATUS "Fuzzing instrumentation enabled") @@ -150,8 +149,7 @@ if (COMPILER_CLANG) endif () endif () -# Adds a Google.Test target binary containing unit tests. -option (ENABLE_TESTS ON) +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" ON) if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") # Only for Linux, x86_64. @@ -251,7 +249,7 @@ if (COMPILER_GCC OR COMPILER_CLANG) endif () # Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc -option(WITH_COVERAGE) +option(WITH_COVERAGE "Profile the resulting binary/binaries") if (WITH_COVERAGE AND COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") @@ -385,7 +383,7 @@ else () endif () # https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE OFF) +option (USE_INCLUDE_WHAT_YOU_USE "Automatically reduce unneeded includes in source code (external tool)" OFF) if (USE_INCLUDE_WHAT_YOU_USE) find_program(IWYU_PATH NAMES include-what-you-use iwyu) diff --git a/cmake_files_footer.md b/cmake_files_footer.md index a35d39cf9d3..21cf7677753 100644 --- a/cmake_files_footer.md +++ b/cmake_files_footer.md @@ -8,25 +8,31 @@ Bad: option (ENABLE_TESTS "Enables testing" OFF) ``` -This description is quite useless as is neither gives the viewer any additional information nor explains the option -purpose. If the option's name is quite self-descriptive, prefer the empty description. +This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. Better: ```cmake -option(ENABLE_TESTS OFF) +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) ``` -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above -the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some +pre-conditions, leave a comment above the `option()` line and explain what it does. +The best way would be linking the docs page (if it exists). The comment is parsed into a separate column (see below). -Even better (default off value is omitted): +Even better: ```cmake -# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). +# implies ${TESTS_ARE_ENABLED} # see tests/CMakeLists.txt for implementation detail. -option(ENABLE_GTEST_TESTS) +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests") +``` + +Note that the default value (`OFF`) can be omitted if you provide a description, e.g. + +``` +option(MYOPTION "My description") ``` ### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 55f9df7887e..4731b12efa2 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -77,7 +77,7 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L100) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | | [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size.. Tradeoff is the inability to debug some source files with e.g. gdb. (empty stack frames and no local variables).". | | [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L387) | `OFF` | | https://github.com/include-what-you-use/include-what-you-use. | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L387) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | | [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | | [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | | [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | @@ -117,7 +117,7 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | | [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | | [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | | Enable fuzzy testing using libfuzzer. | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | Enable fuzzy testing using libfuzzer. | ## Developer's guide for adding new CMake options @@ -128,25 +128,29 @@ Bad: option (ENABLE_TESTS "Enables testing" OFF) ``` -This description is quite useless as is neither gives the viewer any additional information nor explains the option -purpose. If the option's name is quite self-descriptive, prefer the empty description. +This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. Better: ```cmake -option(ENABLE_TESTS OFF) +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) ``` If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). The comment is parsed into a separate column (see below). -Even better (default off value is omitted): +Even better: ```cmake -# Adds the ability to test ClickHouse using Google.Test (would produce another target unit_tests_dbms). # see tests/CMakeLists.txt for implementation detail. -option(ENABLE_GTEST_TESTS) +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests") +``` + +Note that the default value (`OFF`) can be omitted if you provide a description, e.g. + +``` +option(MYOPTION "My description") ``` ### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. From da2a3fffe88665bb5c5e8e5e9546f1955cac4fd5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 16 Sep 2020 13:00:15 +0300 Subject: [PATCH 087/390] fixup --- src/Dictionaries/CacheDictionary.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 29aee9bfc21..cb39dffeb6c 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -822,7 +822,24 @@ void CacheDictionary::waitForCurrentUpdateFinish(UpdateUnitPtr & update_unit_ptr if (update_unit_ptr->current_exception) - std::rethrow_exception(update_unit_ptr->current_exception); + { + // There might have been a single update unit for multiple callers in + // independent threads, and current_exception will be the same for them. + // Don't just rethrow it, because sharing the same exception object + // between multiple threads can lead to weird effects if they decide to + // modify it, for example, by adding some error context. + try + { + std::rethrow_exception(update_unit_ptr->current_exception); + } + catch (...) + { + throw DB::Exception(ErrorCodes::CACHE_DICTIONARY_UPDATE_FAIL, + "Dictionary update failed: {}", + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } + } } void CacheDictionary::tryPushToUpdateQueueOrThrow(UpdateUnitPtr & update_unit_ptr) const From 8be45dd3cc53017a033bc22fba373f47eaf69010 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 13:25:01 +0300 Subject: [PATCH 088/390] Update QueryPipeline::addDelayingPipeline --- src/Processors/QueryPipeline.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 0337f19c2e3..c122dd240f7 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -264,16 +264,22 @@ QueryPipeline QueryPipeline::unitePipelines( void QueryPipeline::addDelayingPipeline(QueryPipeline pipeline) { - pipeline.resize(1); + checkInitializedAndNotCompleted(); + assertBlocksHaveEqualStructure(getHeader(), pipeline.getHeader(), "QueryPipeline"); + + IProcessor::PortNumbers delayed_streams(pipe.numOutputPorts()); + for (size_t i = 0; i < delayed_streams.size(); ++i) + delayed_streams[i] = i; auto * collected_processors = pipe.collected_processors; Pipes pipes; - pipes.emplace_back(QueryPipeline::getPipe(std::move(pipeline))); pipes.emplace_back(std::move(pipe)); + pipes.emplace_back(QueryPipeline::getPipe(std::move(pipeline))); pipe = Pipe::unitePipes(std::move(pipes), collected_processors); - pipe.addTransform(std::make_shared(getHeader(), 2)); + auto processor = std::make_shared(getHeader(), pipe.numOutputPorts(), delayed_streams); + addTransform(std::move(processor)); } void QueryPipeline::setProgressCallback(const ProgressCallback & callback) From b33a5e0d140dc9c9a942d6bc05c02d292143ae80 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 16 Sep 2020 15:32:33 +0300 Subject: [PATCH 089/390] done --- src/Dictionaries/CacheDictionary.inc.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 27064d113e6..1f0bca0d01f 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -148,15 +148,23 @@ void CacheDictionary::getItemsNumberImpl( std::begin(cache_expired_ids), std::end(cache_expired_ids), std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - auto on_cell_updated = [&] (const auto id, const auto cell_idx) + auto on_cell_updated = + [&attribute_array, &cache_not_found_ids, &cache_expired_ids, &out] + (const auto id, const auto cell_idx) { const auto attribute_value = attribute_array[cell_idx]; - for (const size_t row : cache_not_found_ids[id]) - out[row] = static_cast(attribute_value); + auto cache_not_found_it = cache_not_found_ids.find(id); - for (const size_t row : cache_expired_ids[id]) - out[row] = static_cast(attribute_value); + if (cache_not_found_it != cache_not_found_ids.end()) + for (const size_t row : cache_not_found_it->second) + out[row] = static_cast(attribute_value); + + auto cache_expired_it = cache_expired_ids.find(id); + + if (cache_expired_it != cache_expired_ids.end()) + for (const size_t row : cache_expired_it->second) + out[row] = static_cast(attribute_value); }; auto on_id_not_found = [&] (auto, auto) {}; From 8dc3c9b2394c8524a63b1819dda4573146f25dab Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Wed, 16 Sep 2020 18:14:33 +0400 Subject: [PATCH 090/390] Edit and translate. --- .../aggregate-functions/reference/maxmap.md | 9 ++- .../aggregate-functions/reference/minmap.md | 2 +- .../functions/type-conversion-functions.md | 4 +- .../aggregate-functions/reference/maxmap.md | 28 +++++++ .../aggregate-functions/reference/minmap.md | 28 +++++++ .../functions/type-conversion-functions.md | 74 +++++++++++++++++++ 6 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 docs/ru/sql-reference/aggregate-functions/reference/maxmap.md create mode 100644 docs/ru/sql-reference/aggregate-functions/reference/minmap.md diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md index 4dca13ed1b4..ffb36f06e61 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md +++ b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md @@ -6,9 +6,12 @@ toc_priority: 143 Syntax: `maxMap(key, value)` or `maxMap(Tuple(key, value))` -Calculates the maximum from `value` array according to the keys specified in the ‘key’ array. -Passing tuple of keys and values arrays is synonymical to passing two arrays of keys and values. -The number of elements in ‘key’ and ‘value’ must be the same for each row that is totaled. +Calculates the maximum from `value` array according to the keys specified in the `key` array. + +Passing a tuple of keys and value ​​arrays is identical to passing two arrays of keys and values. + +The number of elements in `key` and `value` must be the same for each row that is totaled. + Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. Example: diff --git a/docs/en/sql-reference/aggregate-functions/reference/minmap.md b/docs/en/sql-reference/aggregate-functions/reference/minmap.md index 1b946dea209..9408d0ddfff 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/minmap.md +++ b/docs/en/sql-reference/aggregate-functions/reference/minmap.md @@ -8,7 +8,7 @@ Syntax: `minMap(key, value)` or `minMap(Tuple(key, value))` Calculates the minimum from `value` array according to the keys specified in the `key` array. -Passing tuple of keys and values arrays is a synonym to passing two arrays of keys and values. +Passing a tuple of keys and value ​​arrays is identical to passing two arrays of keys and values. The number of elements in `key` and `value` must be the same for each row that is totaled. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 67361c350c7..515a4a650ce 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -523,7 +523,7 @@ Result: ## parseDateTimeBestEffortUS {#parsedatetimebesteffortUS} -This function is similar to [‘parseDateTimeBestEffort’](#parsedatetimebesteffort), the only difference is that this function prefers US style (`MM/DD/YYYY` etc) in case of ambiguouty. +This function is similar to [‘parseDateTimeBestEffort’](#parsedatetimebesteffort), the only difference is that this function prefers US date format (`MM/DD/YYYY` etc.) in case of ambiguity. **Syntax** @@ -541,7 +541,7 @@ parseDateTimeBestEffortUS(time_string [, time_zone]); - A string containing 9..10 digit [unix timestamp](https://en.wikipedia.org/wiki/Unix_time). - A string with a date and a time component: `YYYYMMDDhhmmss`, `MM/DD/YYYY hh:mm:ss`, `MM-DD-YY hh:mm`, `YYYY-MM-DD hh:mm:ss`, etc. - A string with a date, but no time component: `YYYY`, `YYYYMM`, `YYYY*MM`, `MM/DD/YYYY`, `MM-DD-YY` etc. -- A string with a day and time: `DD`, `DD hh`, `DD hh:mm`. In this case `YYYY-MM` are substituted as `2000-01`. +- A string with a day and time: `DD`, `DD hh`, `DD hh:mm`. In this case, `YYYY-MM` are substituted as `2000-01`. - A string that includes the date and time along with time zone offset information: `YYYY-MM-DD hh:mm:ss ±h:mm`, etc. For example, `2020-12-12 17:36:00 -5:00`. **Returned value** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md b/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md new file mode 100644 index 00000000000..efac771666b --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md @@ -0,0 +1,28 @@ +--- +toc_priority: 143 +--- + +# maxMap {#agg_functions-maxmap} + +Синтаксис: `maxMap(key, value)` or `maxMap(Tuple(key, value))` + +Вычисляет максимальное значение массива `value` в соответствии с ключами, указанными в массиве `key`. + +Передача кортежа ключей и массивов значений идентична передаче двух массивов ключей и значений. + +Количество элементов в параметрах `key` и `value` должно быть одинаковым для каждой суммируемой строки. + +Возвращает кортеж из двух массивов: ключи в отсортированном порядке и значения, рассчитанные для соответствующих ключей. + +Пример: + +``` sql +SELECT maxMap(a, b) +FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1])) +``` + +``` text +┌─maxMap(a, b)──────┐ +│ ([1,2,3],[2,2,1]) │ +└───────────────────┘ +``` diff --git a/docs/ru/sql-reference/aggregate-functions/reference/minmap.md b/docs/ru/sql-reference/aggregate-functions/reference/minmap.md new file mode 100644 index 00000000000..e6def16e583 --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/minmap.md @@ -0,0 +1,28 @@ +--- +toc_priority: 142 +--- + +# minMap {#agg_functions-minmap} + +Синтаксис: `minMap(key, value)` or `minMap(Tuple(key, value))` + +Вычисляет минимальное значение массива `value` в соответствии с ключами, указанными в массиве `key`. + +Передача кортежа ключей и массивов значений идентична передаче двух массивов ключей и значений. + +Количество элементов в параметрах `key` и `value` должно быть одинаковым для каждой суммируемой строки. + +Возвращает кортеж из двух массивов: ключи в отсортированном порядке и значения, рассчитанные для соответствующих ключей. + +Пример: + +``` sql +SELECT minMap(a, b) +FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1])) +``` + +``` text +┌─minMap(a, b)──────┐ +│ ([1,2,3],[2,1,1]) │ +└───────────────────┘ +``` diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index c7d74a9d881..7a57b94c4cd 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -513,6 +513,80 @@ SELECT parseDateTimeBestEffort('10 20:19') - [toDate](#todate) - [toDateTime](#todatetime) +## parseDateTimeBestEffortUS {#parsedatetimebesteffortUS} + +Эта функция похожа на [‘parseDateTimeBestEffort’](#parsedatetimebesteffort), но разница состоит в том, что в она использует американский формат даты (`MM/DD/YYYY` etc.) в случае многозначности. + +**Синтаксис** + +``` sql +parseDateTimeBestEffortUS(time_string [, time_zone]); +``` + +**Параметры** + +- `time_string` — строка, содержащая дату и время для преобразования. [String](../../sql-reference/data-types/string.md). +- `time_zone` — часовой пояс. Функция анализирует `time_string` в соответствии с часовым поясом. [String](../../sql-reference/data-types/string.md). + +**Поддерживаемые нестандартные форматы** + +- Строка, содержащая 9-10 цифр [unix timestamp](https://en.wikipedia.org/wiki/Unix_time). +- Строка, содержащая дату и время: `YYYYMMDDhhmmss`, `MM/DD/YYYY hh:mm:ss`, `MM-DD-YY hh:mm`, `YYYY-MM-DD hh:mm:ss`, etc. +- Строка с датой, но без времени: `YYYY`, `YYYYMM`, `YYYY*MM`, `MM/DD/YYYY`, `MM-DD-YY` etc. +- Строка, содержащая день и время: `DD`, `DD hh`, `DD hh:mm`. В этом случае `YYYY-MM` заменяется на `2000-01`. +- Строка, содержащая дату и время, а также информацию о часовом поясе: `YYYY-MM-DD hh:mm:ss ±h:mm` и т.д. Например, `2020-12-12 17:36:00 -5:00`. + +**Возвращаемое значение** + +- `time_string` преобразован в тип данных `DateTime`. + +**Примеры** + +Запрос: + +``` sql +SELECT parseDateTimeBestEffortUS('09/12/2020 12:12:57') +AS parseDateTimeBestEffortUS; +``` + +Ответ: + +``` text +┌─parseDateTimeBestEffortUS─┐ +│ 2020-09-12 12:12:57 │ +└─────────────────────────——┘ +``` + +Запрос: + +``` sql +SELECT parseDateTimeBestEffortUS('09-12-2020 12:12:57') +AS parseDateTimeBestEffortUS; +``` + +Ответ: + +``` text +┌─parseDateTimeBestEffortUS─┐ +│ 2020-09-12 12:12:57 │ +└─────────────────────────——┘ +``` + +Запрос: + +``` sql +SELECT parseDateTimeBestEffortUS('09.12.2020 12:12:57') +AS parseDateTimeBestEffortUS; +``` + +Ответ: + +``` text +┌─parseDateTimeBestEffortUS─┐ +│ 2020-09-12 12:12:57 │ +└─────────────────────────——┘ +``` + ## toUnixTimestamp64Milli ## toUnixTimestamp64Micro ## toUnixTimestamp64Nano From 8de76709fb5d5eb50f3668c69bea3fd440f63e40 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 17:41:01 +0300 Subject: [PATCH 091/390] Fix tests. --- src/Processors/QueryPlan/CreatingSetsStep.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index f63b657c029..bf2a0aa1929 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -121,6 +121,9 @@ void addCreatingSetsStep( for (auto & [description, set] : subqueries_for_sets) { + if (!set.source) + continue; + auto plan = std::move(set.source); std::string type = (set.join != nullptr) ? "JOIN" : "subquery"; @@ -139,6 +142,12 @@ void addCreatingSetsStep( plans.emplace_back(std::move(plan)); } + if (plans.size() == 1) + { + query_plan = std::move(*plans.front()); + return; + } + auto creating_sets = std::make_unique(std::move(input_streams)); creating_sets->setStepDescription("Create sets before main query execution"); query_plan.unitePlans(std::move(creating_sets), std::move(plans)); From 8a26251f94f7bbeaa21b41e6d930d8458b530d25 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 19:11:16 +0300 Subject: [PATCH 092/390] Fix tests. --- src/Processors/QueryPipeline.cpp | 20 +++++++++++++++++++ src/Processors/QueryPipeline.h | 2 ++ src/Processors/QueryPlan/CreatingSetsStep.cpp | 9 +-------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index c122dd240f7..7f7e2391bde 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -262,6 +262,26 @@ QueryPipeline QueryPipeline::unitePipelines( return pipeline; } + +void QueryPipeline::addCreatingSetsTransform() +{ + pipeline.resize(1); + + auto transform = std::make_shared( + pipeline.getHeader(), + getOutputStream().header, + std::move(subquery_for_set), + network_transfer_limits, + context)); + + InputPort * totals_port = nullptr; + + if (pipe.getTotalsPort()) + totals_port = transform->addTotalsPort(); + + pipe.addTransform(std::move(transform), totals_port, nullptr); +} + void QueryPipeline::addDelayingPipeline(QueryPipeline pipeline) { checkInitializedAndNotCompleted(); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 4be7bbfd573..e2083ddae66 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -89,6 +89,8 @@ public: /// Pipeline must have same header. void addDelayingPipeline(QueryPipeline pipeline); + void addCreatingSetsTransform(); + PipelineExecutorPtr execute(); size_t getNumStreams() const { return pipe.numOutputPorts(); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index bf2a0aa1929..6a7cfc5ced8 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -39,14 +39,7 @@ CreatingSetStep::CreatingSetStep( void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) { - pipeline.resize(1); - pipeline.addTransform( - std::make_shared( - pipeline.getHeader(), - getOutputStream().header, - std::move(subquery_for_set), - network_transfer_limits, - context)); + pipeline.addCreatingSetsTransform(); } void CreatingSetStep::describeActions(FormatSettings & settings) const From 2f7ab2bca4715b0c2f13547b5ee7a0e26b8f48df Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 19:30:48 +0300 Subject: [PATCH 093/390] Fix tests. --- src/Processors/QueryPipeline.cpp | 6 +++--- src/Processors/QueryPipeline.h | 4 +++- src/Processors/QueryPlan/CreatingSetsStep.cpp | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 7f7e2391bde..bef0c0a3047 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -263,15 +263,15 @@ QueryPipeline QueryPipeline::unitePipelines( } -void QueryPipeline::addCreatingSetsTransform() +void QueryPipeline::addCreatingSetsTransform(SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context) { - pipeline.resize(1); + resize(1); auto transform = std::make_shared( pipeline.getHeader(), getOutputStream().header, std::move(subquery_for_set), - network_transfer_limits, + limits, context)); InputPort * totals_port = nullptr; diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index e2083ddae66..66947bc9fa5 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -26,6 +26,8 @@ class QueryPlan; struct SubqueryForSet; using SubqueriesForSets = std::unordered_map; +struct SizeLimits; + class QueryPipeline { public: @@ -89,7 +91,7 @@ public: /// Pipeline must have same header. void addDelayingPipeline(QueryPipeline pipeline); - void addCreatingSetsTransform(); + void addCreatingSetsTransform(SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context); PipelineExecutorPtr execute(); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 6a7cfc5ced8..4de11d0791d 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -39,7 +39,7 @@ CreatingSetStep::CreatingSetStep( void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) { - pipeline.addCreatingSetsTransform(); + pipeline.addCreatingSetsTransform(std::move(subquery_for_set), network_transfer_limits, context); } void CreatingSetStep::describeActions(FormatSettings & settings) const From 73ad5057357e4e2e41c52fad9fda3308b05aed43 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 19:38:33 +0300 Subject: [PATCH 094/390] Fix tests. --- src/Processors/QueryPipeline.cpp | 4 ++-- src/Processors/QueryPipeline.h | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index bef0c0a3047..598bb0c8053 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -263,13 +263,13 @@ QueryPipeline QueryPipeline::unitePipelines( } -void QueryPipeline::addCreatingSetsTransform(SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context) +void QueryPipeline::addCreatingSetsTransform(const Block & res_header, SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context) { resize(1); auto transform = std::make_shared( pipeline.getHeader(), - getOutputStream().header, + res_header, std::move(subquery_for_set), limits, context)); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 66947bc9fa5..2a318a53b76 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -91,7 +91,7 @@ public: /// Pipeline must have same header. void addDelayingPipeline(QueryPipeline pipeline); - void addCreatingSetsTransform(SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context); + void addCreatingSetsTransform(const Block & res_header, SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context); PipelineExecutorPtr execute(); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 4de11d0791d..27ba3153151 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -39,7 +39,7 @@ CreatingSetStep::CreatingSetStep( void CreatingSetStep::transformPipeline(QueryPipeline & pipeline) { - pipeline.addCreatingSetsTransform(std::move(subquery_for_set), network_transfer_limits, context); + pipeline.addCreatingSetsTransform(getOutputStream().header, std::move(subquery_for_set), network_transfer_limits, context); } void CreatingSetStep::describeActions(FormatSettings & settings) const From fe1056579f0563c7b427b2baf2bae59fd568430f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 19:45:56 +0300 Subject: [PATCH 095/390] Fix tests. --- src/Processors/QueryPipeline.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 598bb0c8053..c8218023a57 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -268,11 +268,11 @@ void QueryPipeline::addCreatingSetsTransform(const Block & res_header, SubqueryF resize(1); auto transform = std::make_shared( - pipeline.getHeader(), + getHeader(), res_header, std::move(subquery_for_set), limits, - context)); + context); InputPort * totals_port = nullptr; From e1736584225ee8c36c7eb4ac6eba024c470850ce Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 16 Sep 2020 19:53:58 +0300 Subject: [PATCH 096/390] Fix integration test --- .../test_replicated_merge_tree_config/test.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_config/test.py b/tests/integration/test_replicated_merge_tree_config/test.py index bcf6517782d..e3bc88b155f 100644 --- a/tests/integration/test_replicated_merge_tree_config/test.py +++ b/tests/integration/test_replicated_merge_tree_config/test.py @@ -1,14 +1,14 @@ import pytest from helpers.cluster import ClickHouseCluster +import logging + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/config.xml"], with_zookeeper=True) @pytest.fixture(scope="module") -def cluster(): +def start_cluster(): try: - cluster = ClickHouseCluster(__file__) - cluster.add_instance( - "node", config_dir="configs", with_zookeeper=True, - ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -19,19 +19,18 @@ def cluster(): @pytest.fixture(autouse=True) -def drop_table(cluster): +def drop_table(start_cluster): yield for node in cluster.instances.values(): node.query("DROP TABLE IF EXISTS test1") node.query("DROP TABLE IF EXISTS test2") -def test_replicated_merge_tree_settings(cluster): - node = cluster.instances["node"] +def test_replicated_merge_tree_settings(start_cluster): node.query("CREATE TABLE test1 (id Int64) ENGINE MergeTree ORDER BY id") node.query( "CREATE TABLE test2 (id Int64) ENGINE ReplicatedMergeTree('/clickhouse/test', 'test') ORDER BY id" ) - assert node.query("SHOW CREATE test1").endswith("100") - assert node.query("SHOW CREATE test2").endswith("200") + assert node.query("SHOW CREATE test1").strip().endswith("100") + assert node.query("SHOW CREATE test2").strip().endswith("200") From d0debd51c0263fd00cc49d391cf499c733ad2c7a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Sep 2020 20:00:44 +0300 Subject: [PATCH 097/390] Add sorting to fix test. --- tests/queries/0_stateless/00853_join_with_nulls_crash.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql index eb64ed29ffe..464ddbb1990 100644 --- a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql +++ b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql @@ -21,14 +21,14 @@ SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2 ALL FULL JOIN ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2 ON s1.other = s2.other -ORDER BY s2.other DESC; +ORDER BY s2.other DESC, count_a; SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2.other) FROM ( SELECT other, count() AS count_a FROM table_a GROUP BY other ) s1 ALL FULL JOIN ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2 USING other -ORDER BY s2.other DESC; +ORDER BY s2.other DESC, count_a; SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), toTypeName(s2.something) FROM ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) s1 From bf9feb68659fc3c04022b35f751f4c80249694ed Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 16 Sep 2020 22:07:58 -0400 Subject: [PATCH 098/390] Removing usage of time.sleep in tests/integration/test_distributed_over_live_view/test.py --- .../test_distributed_over_live_view/test.py | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_distributed_over_live_view/test.py b/tests/integration/test_distributed_over_live_view/test.py index 46c0bada535..ed6b8be23df 100644 --- a/tests/integration/test_distributed_over_live_view/test.py +++ b/tests/integration/test_distributed_over_live_view/test.py @@ -60,6 +60,13 @@ def started_cluster(): finally: cluster.shutdown() +def poll_query(node, query, expected, timeout): + """Repeatedly execute query until either expected result is returned or timeout occurs. + """ + start_time = time.time() + while node.query(query) != expected and time.time() - start_time < timeout: + pass + assert node.query(query) == expected @pytest.mark.parametrize("node", NODES.values()[:1]) @pytest.mark.parametrize("source", ["lv_over_distributed_table"]) @@ -69,6 +76,8 @@ class TestLiveViewOverDistributedSuite: node0, node1 = NODES.values() select_query = "SELECT * FROM distributed_over_lv ORDER BY node, key FORMAT CSV" + select_query_dist_table = "SELECT * FROM distributed_table ORDER BY node, key FORMAT CSV" + select_count_query = "SELECT count() FROM distributed_over_lv" with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: @@ -91,7 +100,17 @@ class TestLiveViewOverDistributedSuite: client1.expect(prompt) client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, select_count_query, "7\n", timeout=60) + print("\n--DEBUG1--") + print(select_query) + print(node0.query(select_query)) + print("---------") + print("\n--DEBUG2--") + print(select_query_dist_table) + print(node0.query(select_query_dist_table)) + print("---------") + client1.send(select_query) client1.expect('"node1",0,0') client1.expect('"node1",1,1') @@ -107,6 +126,7 @@ class TestLiveViewOverDistributedSuite: node0, node1 = NODES.values() select_query = "SELECT * FROM distributed_over_lv ORDER BY key, node FORMAT CSV" + select_count_query = "SELECT count() FROM distributed_over_lv" with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: @@ -129,7 +149,9 @@ class TestLiveViewOverDistributedSuite: client1.expect(prompt) client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, select_count_query, "7\n", timeout=60) + client1.send(select_query) client1.expect('"node1",0,0') client1.expect('"node2",0,10') @@ -163,7 +185,9 @@ class TestLiveViewOverDistributedSuite: client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, select_query, '"node1",3\n"node2",21\n', timeout=60) + client1.send(select_query) client1.expect('"node1",3') client1.expect('"node2",21') @@ -173,7 +197,9 @@ class TestLiveViewOverDistributedSuite: client1.expect(prompt) client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, select_query, '"node1",12\n"node2",21\n', timeout=60) + client1.send(select_query) client1.expect('"node1",12') client1.expect('"node2",21') @@ -202,7 +228,9 @@ class TestLiveViewOverDistributedSuite: client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, "SELECT count() FROM (%s)" % select_query.rsplit("FORMAT")[0], "3\n", timeout=60) + client1.send(select_query) client1.expect('0,10') client1.expect('1,12') @@ -211,7 +239,9 @@ class TestLiveViewOverDistributedSuite: client2.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 3, 3)") client2.expect(prompt) - time.sleep(2) + + poll_query(node0, "SELECT count() FROM (%s)" % select_query.rsplit("FORMAT")[0], "4\n", timeout=60) + client1.send(select_query) client1.expect('0,10') client1.expect('1,15') @@ -240,18 +270,9 @@ class TestLiveViewOverDistributedSuite: client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") client2.expect(prompt) - time.sleep(2) - - client1.send("SELECT sum(value) FROM distributed_over_lv") - client1.expect(r"24" + end_of_block) - client1.expect(prompt) + poll_query(node0, "SELECT sum(value) FROM distributed_over_lv", "24\n", timeout=60) client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3), ('node1', 4, 4)") client2.expect(prompt) - time.sleep(2) - - client1.send("SELECT sum(value) FROM distributed_over_lv") - client1.expect(r"31" + end_of_block) - client1.expect(prompt) - + poll_query(node0, "SELECT sum(value) FROM distributed_over_lv", "31\n", timeout=60) From 73847331444e4cade1709345f5ad2ec6f863dddf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 09:01:35 +0300 Subject: [PATCH 099/390] Fix style. --- src/Processors/QueryPlan/CreatingSetsStep.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 27ba3153151..780e8493b40 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + static ITransformingStep::Traits getTraits() { return ITransformingStep::Traits From 339521fadbfc3a82a8a0b3f2fae5ac1ec48ca7b1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Sep 2020 11:48:27 +0300 Subject: [PATCH 100/390] Update test.py --- tests/integration/test_replicated_merge_tree_config/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_config/test.py b/tests/integration/test_replicated_merge_tree_config/test.py index e3bc88b155f..2a7725960bf 100644 --- a/tests/integration/test_replicated_merge_tree_config/test.py +++ b/tests/integration/test_replicated_merge_tree_config/test.py @@ -32,5 +32,5 @@ def test_replicated_merge_tree_settings(start_cluster): "CREATE TABLE test2 (id Int64) ENGINE ReplicatedMergeTree('/clickhouse/test', 'test') ORDER BY id" ) - assert node.query("SHOW CREATE test1").strip().endswith("100") - assert node.query("SHOW CREATE test2").strip().endswith("200") + assert "index_granularity = 100" in node.query("SHOW CREATE test1") + assert "index_granularity = 200" in node.query("SHOW CREATE test2") From 72c68a8746bc22ec8e258fcd2e177833c8b867d7 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 13:05:03 +0300 Subject: [PATCH 101/390] boop the CI From 8bce20076c0e2b38b5957cd9bc3021516030132a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 15:39:37 +0300 Subject: [PATCH 102/390] Only mlock code segment --- programs/server/Server.cpp | 48 ++++++++++++++++++---------------- src/Common/remapExecutable.cpp | 47 ++++++++++++++++----------------- src/Common/remapExecutable.h | 3 +++ 3 files changed, 52 insertions(+), 46 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 56778b8dd69..2baca4c7379 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -90,6 +90,23 @@ namespace CurrentMetrics extern const Metric MemoryTracking; } + +int mainEntryClickHouseServer(int argc, char ** argv) +{ + DB::Server app; + try + { + return app.run(argc, argv); + } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + return code ? code : 1; + } +} + + namespace { @@ -317,11 +334,16 @@ int Server::main(const std::vector & /*args*/) { if (hasLinuxCapability(CAP_IPC_LOCK)) { - LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); - if (0 != mlockall(MCL_CURRENT)) - LOG_WARNING(log, "Failed mlockall: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); + /// Get the memory area with (current) code segment. + /// It's better to lock only the code segment instead of calling "mlockall", + /// because otherwise debug info will be also locked in memory, and it can be huge. + auto [addr, len] = getMappedArea(reinterpret_cast(mainEntryClickHouseServer)); + + LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds."); + if (0 != mlock(addr, len)) + LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); else - LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed"); + LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len)); } else { @@ -1132,21 +1154,3 @@ int Server::main(const std::vector & /*args*/) return Application::EXIT_OK; } } - -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wmissing-declarations" - -int mainEntryClickHouseServer(int argc, char ** argv) -{ - DB::Server app; - try - { - return app.run(argc, argv); - } - catch (...) - { - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - auto code = DB::getCurrentExceptionCode(); - return code ? code : 1; - } -} diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp index 13bce459022..35d0338f9ce 100644 --- a/src/Common/remapExecutable.cpp +++ b/src/Common/remapExecutable.cpp @@ -48,30 +48,6 @@ uintptr_t readAddressHex(DB::ReadBuffer & in) } -/** Find the address and size of the mapped memory region pointed by ptr. - */ -std::pair getMappedArea(void * ptr) -{ - using namespace DB; - - uintptr_t uintptr = reinterpret_cast(ptr); - ReadBufferFromFile in("/proc/self/maps"); - - while (!in.eof()) - { - uintptr_t begin = readAddressHex(in); - assertChar('-', in); - uintptr_t end = readAddressHex(in); - skipToNextLineOrEOF(in); - - if (begin <= uintptr && uintptr < end) - return {reinterpret_cast(begin), end - begin}; - } - - throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR); -} - - __attribute__((__noinline__)) int64_t our_syscall(...) { __asm__ __volatile__ (R"( @@ -181,6 +157,28 @@ __attribute__((__noinline__)) void remapToHugeStep1(void * begin, size_t size) } +std::pair getMappedArea(void * ptr) +{ + using namespace DB; + + uintptr_t uintptr = reinterpret_cast(ptr); + ReadBufferFromFile in("/proc/self/maps"); + + while (!in.eof()) + { + uintptr_t begin = readAddressHex(in); + assertChar('-', in); + uintptr_t end = readAddressHex(in); + skipToNextLineOrEOF(in); + + if (begin <= uintptr && uintptr < end) + return {reinterpret_cast(begin), end - begin}; + } + + throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR); +} + + void remapExecutable() { auto [begin, size] = getMappedArea(reinterpret_cast(remapExecutable)); @@ -195,6 +193,7 @@ namespace DB { void remapExecutable() {} +std::pair getMappedArea(void * ptr) { return {}; } } diff --git a/src/Common/remapExecutable.h b/src/Common/remapExecutable.h index 7acb61f13bd..af0a96d5248 100644 --- a/src/Common/remapExecutable.h +++ b/src/Common/remapExecutable.h @@ -4,4 +4,7 @@ namespace DB /// This function tries to reallocate the code of the running program in a more efficient way. void remapExecutable(); +/// Find the address and size of the mapped memory region pointed by ptr. +std::pair getMappedArea(void * ptr); + } From 068e8576b7a930c4d7fdd9420a17e69c8b93eaaf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 15:53:52 +0300 Subject: [PATCH 103/390] Corrections --- programs/server/Server.cpp | 1 + src/Common/getMappedArea.cpp | 80 ++++++++++++++++++++++++++++++++++ src/Common/getMappedArea.h | 10 +++++ src/Common/remapExecutable.cpp | 51 ++-------------------- src/Common/remapExecutable.h | 3 -- 5 files changed, 94 insertions(+), 51 deletions(-) create mode 100644 src/Common/getMappedArea.cpp create mode 100644 src/Common/getMappedArea.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 2baca4c7379..0bb64a29a09 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp new file mode 100644 index 00000000000..6817bea8cb1 --- /dev/null +++ b/src/Common/getMappedArea.cpp @@ -0,0 +1,80 @@ +#if defined(__linux__) + +#include +#include +#include +#include +#include + +#include "getMappedArea.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + + +namespace +{ + +uintptr_t readAddressHex(DB::ReadBuffer & in) +{ + uintptr_t res = 0; + while (!in.eof()) + { + if (isHexDigit(*in.position())) + { + res *= 16; + res += unhex(*in.position()); + ++in.position(); + } + else + break; + } + return res; +} + +} + +std::pair getMappedArea(void * ptr) +{ + using namespace DB; + + uintptr_t uintptr = reinterpret_cast(ptr); + ReadBufferFromFile in("/proc/self/maps"); + + while (!in.eof()) + { + uintptr_t begin = readAddressHex(in); + assertChar('-', in); + uintptr_t end = readAddressHex(in); + skipToNextLineOrEOF(in); + + if (begin <= uintptr && uintptr < end) + return {reinterpret_cast(begin), end - begin}; + } + + throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR); +} + +} + +#else + +namespace DB +{ + +std::pair getMappedArea(void * ptr) +{ + throw Exception("The function getMappedArea is implemented only for Linux", ErrorCodes::NOT_IMPLEMENTED); +} + +} + +#endif + diff --git a/src/Common/getMappedArea.h b/src/Common/getMappedArea.h new file mode 100644 index 00000000000..7d120f0d9a7 --- /dev/null +++ b/src/Common/getMappedArea.h @@ -0,0 +1,10 @@ +#include + +namespace DB +{ + +/// Find the address and size of the mapped memory region pointed by ptr. +/// Throw exception if not found. +std::pair getMappedArea(void * ptr); + +} diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp index 35d0338f9ce..5418290b24f 100644 --- a/src/Common/remapExecutable.cpp +++ b/src/Common/remapExecutable.cpp @@ -2,17 +2,14 @@ #include #include +#include #include #include -#include - -#include -#include +#include #include -#include -#include +#include #include "remapExecutable.h" @@ -22,7 +19,6 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int CANNOT_ALLOCATE_MEMORY; } @@ -30,24 +26,6 @@ namespace ErrorCodes namespace { -uintptr_t readAddressHex(DB::ReadBuffer & in) -{ - uintptr_t res = 0; - while (!in.eof()) - { - if (isHexDigit(*in.position())) - { - res *= 16; - res += unhex(*in.position()); - ++in.position(); - } - else - break; - } - return res; -} - - __attribute__((__noinline__)) int64_t our_syscall(...) { __asm__ __volatile__ (R"( @@ -157,28 +135,6 @@ __attribute__((__noinline__)) void remapToHugeStep1(void * begin, size_t size) } -std::pair getMappedArea(void * ptr) -{ - using namespace DB; - - uintptr_t uintptr = reinterpret_cast(ptr); - ReadBufferFromFile in("/proc/self/maps"); - - while (!in.eof()) - { - uintptr_t begin = readAddressHex(in); - assertChar('-', in); - uintptr_t end = readAddressHex(in); - skipToNextLineOrEOF(in); - - if (begin <= uintptr && uintptr < end) - return {reinterpret_cast(begin), end - begin}; - } - - throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR); -} - - void remapExecutable() { auto [begin, size] = getMappedArea(reinterpret_cast(remapExecutable)); @@ -193,7 +149,6 @@ namespace DB { void remapExecutable() {} -std::pair getMappedArea(void * ptr) { return {}; } } diff --git a/src/Common/remapExecutable.h b/src/Common/remapExecutable.h index af0a96d5248..7acb61f13bd 100644 --- a/src/Common/remapExecutable.h +++ b/src/Common/remapExecutable.h @@ -4,7 +4,4 @@ namespace DB /// This function tries to reallocate the code of the running program in a more efficient way. void remapExecutable(); -/// Find the address and size of the mapped memory region pointed by ptr. -std::pair getMappedArea(void * ptr); - } From c12fa26fe344934aafc4eb8c2f6e43501385ec6b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sun, 23 Aug 2020 23:59:21 +0300 Subject: [PATCH 104/390] kerberized kafka test works --- .../docker_compose_kerberized_kafka.yml | 56 +++++++ tests/integration/helpers/cluster.py | 46 +++++- tests/integration/tskk/Dockerfile | 11 ++ tests/integration/tskk/__init__.py | 0 .../format_schemas/kafka.proto | 6 + .../format_schemas/template_row.format | 1 + .../clickhouse_path/format_schemas/test.avsc | 11 ++ .../clickhouse_path/format_schemas/test.capnp | 10 ++ .../clickhouse_path/format_schemas/test.proto | 9 ++ tests/integration/tskk/configs/kafka.xml | 26 ++++ tests/integration/tskk/configs/log_conf.xml | 11 ++ tests/integration/tskk/configs/users.xml | 25 ++++ tests/integration/tskk/kafka_pb2.py | 76 ++++++++++ .../integration/tskk/kerberos_image_config.sh | 137 ++++++++++++++++++ .../integration/tskk/secrets/broker_jaas.conf | 15 ++ tests/integration/tskk/secrets/krb.conf | 22 +++ .../tskk/secrets/zookeeper_jaas.conf | 14 ++ tests/integration/tskk/test.py | 137 ++++++++++++++++++ .../tskk/test_kafka_json.reference | 50 +++++++ .../tskk/test_kafka_virtual1.reference | 50 +++++++ .../tskk/test_kafka_virtual2.reference | 50 +++++++ 21 files changed, 761 insertions(+), 2 deletions(-) create mode 100644 docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml create mode 100644 tests/integration/tskk/Dockerfile create mode 100644 tests/integration/tskk/__init__.py create mode 100755 tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto create mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/template_row.format create mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.avsc create mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.capnp create mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.proto create mode 100644 tests/integration/tskk/configs/kafka.xml create mode 100644 tests/integration/tskk/configs/log_conf.xml create mode 100644 tests/integration/tskk/configs/users.xml create mode 100644 tests/integration/tskk/kafka_pb2.py create mode 100644 tests/integration/tskk/kerberos_image_config.sh create mode 100644 tests/integration/tskk/secrets/broker_jaas.conf create mode 100644 tests/integration/tskk/secrets/krb.conf create mode 100644 tests/integration/tskk/secrets/zookeeper_jaas.conf create mode 100644 tests/integration/tskk/test.py create mode 100644 tests/integration/tskk/test_kafka_json.reference create mode 100644 tests/integration/tskk/test_kafka_virtual1.reference create mode 100644 tests/integration/tskk/test_kafka_virtual2.reference diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml new file mode 100644 index 00000000000..f06e26c76cb --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -0,0 +1,56 @@ +version: '2.3' + +services: + kafka_kerberized_zookeeper: + image: confluentinc/cp-zookeeper:5.2.0 + # restart: always + hostname: kafka_kerberized_zookeeper + environment: + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVERS: "kafka_kerberized_zookeeper:2888:3888" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dsun.security.krb5.debug=true" + volumes: + - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/etc/kafka/secrets + - /dev/urandom:/dev/random + depends_on: + - kafka_kerberos + security_opt: + - label:disable + + kerberized_kafka1: + image: confluentinc/cp-kafka:5.2.0 + # restart: always + hostname: kerberized_kafka1 + ports: + - "9092:9092" + - "9093:9093" + environment: + KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://:9093 + KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:9093 + # KAFKA_LISTENERS: INSIDE://kerberized_kafka1:9092,OUTSIDE://kerberized_kafka1:19092 + # KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:9092,OUTSIDE://kerberized_kafka1:19092 + KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: GSSAPI + KAFKA_SASL_ENABLED_MECHANISMS: GSSAPI + KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: OUTSIDE:SASL_PLAINTEXT,UNSECURED_OUTSIDE:PLAINTEXT,UNSECURED_INSIDE:PLAINTEXT, + KAFKA_INTER_BROKER_LISTENER_NAME: OUTSIDE + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: "kafka_kerberized_zookeeper:2181" + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true" + volumes: + - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/etc/kafka/secrets + depends_on: + - kafka_kerberized_zookeeper + - kafka_kerberos + security_opt: + - label:disable + + kafka_kerberos: + build: /home/ilejn/projects/ClickHouse/tests/integration/tskk + hostname: kafka_kerberos + volumes: + - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/tmp/keytab + ports: [88, 749] diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index f5d9290a17e..ab038bc702f 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -125,6 +125,7 @@ class ClickHouseCluster: self.base_zookeeper_cmd = None self.base_mysql_cmd = [] self.base_kafka_cmd = [] + self.base_kerberized_kafka_cmd = [] self.base_rabbitmq_cmd = [] self.base_cassandra_cmd = [] self.pre_zookeeper_commands = [] @@ -133,6 +134,7 @@ class ClickHouseCluster: self.with_mysql = False self.with_postgres = False self.with_kafka = False + self.with_kerberized_kafka = False self.with_rabbitmq = False self.with_odbc_drivers = False self.with_hdfs = False @@ -169,7 +171,7 @@ class ClickHouseCluster: def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None, macros=None, - with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, + with_zookeeper=False, with_mysql=False, with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, @@ -207,6 +209,7 @@ class ClickHouseCluster: zookeeper_config_path=self.zookeeper_config_path, with_mysql=with_mysql, with_kafka=with_kafka, + with_kerberized_kafka=with_kerberized_kafka, with_rabbitmq=with_rabbitmq, with_mongo=with_mongo, with_redis=with_redis, @@ -290,6 +293,13 @@ class ClickHouseCluster: p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] cmds.append(self.base_kafka_cmd) + if with_kerberized_kafka and not self.with_kerberized_kafka: + self.with_kerberized_kafka = True + self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')]) + self.base_kerberized_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', + self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')] + cmds.append(self.base_kerberized_kafka_cmd) + if with_rabbitmq and not self.with_rabbitmq: self.with_rabbitmq = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]) @@ -608,6 +618,13 @@ class ClickHouseCluster: self.kafka_docker_id = self.get_instance_docker_id('kafka1') self.wait_schema_registry_to_start(120) + if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: + env = os.environ.copy() + self.kerberized_kafka_instance_path = instance.path + env['KERBERIZED_KAFKA_DIR'] = self.kerberized_kafka_instance_path + '/' + subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=env) + self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') + if self.with_rabbitmq and self.base_rabbitmq_cmd: subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) self.rabbitmq_docker_id = self.get_instance_docker_id('rabbitmq1') @@ -778,9 +795,12 @@ services: - {instance_config_dir}:/etc/clickhouse-server/ - {db_dir}:/var/lib/clickhouse/ - {logs_dir}:/var/log/clickhouse-server/ + - /etc/passwd:/etc/passwd:ro {binary_volume} {odbc_bridge_volume} {odbc_ini_path} + {keytab_path} + {krb5_conf} entrypoint: {entrypoint_cmd} tmpfs: {tmpfs} cap_add: @@ -810,7 +830,7 @@ class ClickHouseInstance: def __init__( self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, custom_dictionaries, - macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, + macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_kerberized_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, @@ -838,6 +858,7 @@ class ClickHouseInstance: self.with_mysql = with_mysql self.with_kafka = with_kafka + self.with_kerberized_kafka = with_kerberized_kafka self.with_rabbitmq = with_rabbitmq self.with_mongo = with_mongo self.with_redis = with_redis @@ -853,6 +874,13 @@ class ClickHouseInstance: else: self.odbc_ini_path = "" + if with_kerberized_kafka: + self.keytab_path = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets:/tmp/keytab" + self.krb5_conf = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets/krb.conf:/etc/krb5.conf:ro" + else: + self.keytab_path = "" + self.krb5_conf = "" + self.docker_client = None self.ip_address = None self.client = None @@ -1182,6 +1210,15 @@ class ClickHouseInstance: if self.with_zookeeper: shutil.copy(self.zookeeper_config_path, conf_d_dir) + if self.with_kerberized_kafka: + # shutil.copytree(p.abspath(p.join(self.base_dir, 'secrets')), p.abspath(p.join(self.path, 'secrets'))) + secrets_dir = p.abspath(p.join(self.custom_config_dir, os.pardir, 'secrets')) + distutils.dir_util.copy_tree(secrets_dir, p.abspath(p.join(self.path, 'secrets'))) + + # Copy config dir + if self.custom_config_dir: + distutils.dir_util.copy_tree(self.custom_config_dir, configs_dir) + # Copy config.d configs print "Copy custom test config files {} to {}".format(self.custom_main_config_paths, self.config_d_dir) for path in self.custom_main_config_paths: @@ -1217,6 +1254,9 @@ class ClickHouseInstance: depends_on.append("kafka1") depends_on.append("schema-registry") + if self.with_kerberized_kafka: + depends_on.append("kerberized_kafka1") + if self.with_rabbitmq: depends_on.append("rabbitmq1") @@ -1280,6 +1320,8 @@ class ClickHouseInstance: user=os.getuid(), env_file=env_file, odbc_ini_path=odbc_ini_path, + keytab_path=self.keytab_path, + krb5_conf=self.krb5_conf, entrypoint_cmd=entrypoint_cmd, networks=networks, app_net=app_net, diff --git a/tests/integration/tskk/Dockerfile b/tests/integration/tskk/Dockerfile new file mode 100644 index 00000000000..2459b6980ce --- /dev/null +++ b/tests/integration/tskk/Dockerfile @@ -0,0 +1,11 @@ +FROM centos:6.6 + +# based on confluent kerberos, which is not provided as image any more + +RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation + +EXPOSE 88 749 + +ADD ./kerberos_image_config.sh /config.sh + +ENTRYPOINT ["/bin/bash", "/config.sh"] diff --git a/tests/integration/tskk/__init__.py b/tests/integration/tskk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto b/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto new file mode 100755 index 00000000000..96b24be4938 --- /dev/null +++ b/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message KeyValuePair { + uint64 key = 1; + string value = 2; +} \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format b/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format new file mode 100644 index 00000000000..c910aa8589e --- /dev/null +++ b/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format @@ -0,0 +1 @@ +(id = ${id:Escaped}, blockNo = ${blockNo:Escaped}, val1 = ${val1:CSV}, val2 = ${val2:Escaped}, val3 = ${val3:Escaped}) \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc b/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc new file mode 100644 index 00000000000..caf693313de --- /dev/null +++ b/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "blockNo", "type": "int"}, + {"name": "val1", "type": "string"}, + {"name": "val2", "type": "float"}, + {"name": "val3", "type": "int"} + ] + } \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp b/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp new file mode 100644 index 00000000000..44f1961205b --- /dev/null +++ b/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp @@ -0,0 +1,10 @@ +@0x99f75f775fe63dae; + +struct TestRecordStruct +{ + id @0 : Int64; + blockNo @1 : UInt16; + val1 @2 : Text; + val2 @3 : Float32; + val3 @4 : UInt8; +} \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.proto b/tests/integration/tskk/clickhouse_path/format_schemas/test.proto new file mode 100644 index 00000000000..cabdff04a78 --- /dev/null +++ b/tests/integration/tskk/clickhouse_path/format_schemas/test.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +message TestMessage { + int64 id = 1; + uint32 blockNo = 2; + string val1 = 3; + float val2 = 4; + uint32 val3 = 5; +}; diff --git a/tests/integration/tskk/configs/kafka.xml b/tests/integration/tskk/configs/kafka.xml new file mode 100644 index 00000000000..f44ec0782aa --- /dev/null +++ b/tests/integration/tskk/configs/kafka.xml @@ -0,0 +1,26 @@ + + + earliest + + SASL_PLAINTEXT + GSSAPI + kafka + /tmp/keytab/clickhouse.keytab + kafkauser/instance@TEST.CONFLUENT.IO + security + false + + + + + 300 + + 6000 + + diff --git a/tests/integration/tskk/configs/log_conf.xml b/tests/integration/tskk/configs/log_conf.xml new file mode 100644 index 00000000000..95466269afe --- /dev/null +++ b/tests/integration/tskk/configs/log_conf.xml @@ -0,0 +1,11 @@ + + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + \ No newline at end of file diff --git a/tests/integration/tskk/configs/users.xml b/tests/integration/tskk/configs/users.xml new file mode 100644 index 00000000000..246e6b069ef --- /dev/null +++ b/tests/integration/tskk/configs/users.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/tests/integration/tskk/kafka_pb2.py b/tests/integration/tskk/kafka_pb2.py new file mode 100644 index 00000000000..79890682125 --- /dev/null +++ b/tests/integration/tskk/kafka_pb2.py @@ -0,0 +1,76 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: clickhouse_path/format_schemas/kafka.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='clickhouse_path/format_schemas/kafka.proto', + package='', + syntax='proto3', + serialized_pb=_b('\n*clickhouse_path/format_schemas/kafka.proto\"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_KEYVALUEPAIR = _descriptor.Descriptor( + name='KeyValuePair', + full_name='KeyValuePair', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='KeyValuePair.key', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='KeyValuePair.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=46, + serialized_end=88, +) + +DESCRIPTOR.message_types_by_name['KeyValuePair'] = _KEYVALUEPAIR + +KeyValuePair = _reflection.GeneratedProtocolMessageType('KeyValuePair', (_message.Message,), dict( + DESCRIPTOR = _KEYVALUEPAIR, + __module__ = 'clickhouse_path.format_schemas.kafka_pb2' + # @@protoc_insertion_point(class_scope:KeyValuePair) + )) +_sym_db.RegisterMessage(KeyValuePair) + + +# @@protoc_insertion_point(module_scope) diff --git a/tests/integration/tskk/kerberos_image_config.sh b/tests/integration/tskk/kerberos_image_config.sh new file mode 100644 index 00000000000..7b7d61312b2 --- /dev/null +++ b/tests/integration/tskk/kerberos_image_config.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# based on confluent kerberos, which is not provided as image any more + +[[ "TRACE" ]] && set -x + +: ${REALM:=TEST.CONFLUENT.IO} +: ${DOMAIN_REALM:=test.confluent.io} +: ${KERB_MASTER_KEY:=masterkey} +: ${KERB_ADMIN_USER:=admin} +: ${KERB_ADMIN_PASS:=admin} + +create_config() { + : ${KDC_ADDRESS:=$(hostname -f)} + + cat>/etc/krb5.conf</var/kerberos/krb5kdc/kdc.conf< /var/kerberos/krb5kdc/kadm5.acl +} + +create_keytabs() { + + kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO" + + kadmin.local -q "addprinc -randkey kafka/kerberized_kafka1@TEST.CONFLUENT.IO" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/kerberized_kafka1@TEST.CONFLUENT.IO" + # kadmin.local -q "addprinc -randkey kafka/localhost@TEST.CONFLUENT.IO" + # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/localhost@TEST.CONFLUENT.IO" + + kadmin.local -q "addprinc -randkey zkclient@TEST.CONFLUENT.IO" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/zkclient.keytab zkclient@TEST.CONFLUENT.IO" + + # kadmin.local -q "addprinc -randkey kafkauser@TEST.CONFLUENT.IO" + # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser@TEST.CONFLUENT.IO" + + kadmin.local -q "addprinc -randkey kafkauser/instance@TEST.CONFLUENT.IO" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser/instance@TEST.CONFLUENT.IO" + + chmod g+r /tmp/keytab/clickhouse.keytab + +} + +main() { + + if [ ! -f /kerberos_initialized ]; then + create_config + create_db + create_admin_user + start_kdc + + touch /kerberos_initialized + fi + + if [ ! -f /var/kerberos/krb5kdc/principal ]; then + while true; do sleep 1000; done + else + start_kdc + create_keytabs + tail -F /var/log/kerberos/krb5kdc.log + fi + +} + +[[ "$0" == "$BASH_SOURCE" ]] && main "$@" diff --git a/tests/integration/tskk/secrets/broker_jaas.conf b/tests/integration/tskk/secrets/broker_jaas.conf new file mode 100644 index 00000000000..db119698d62 --- /dev/null +++ b/tests/integration/tskk/secrets/broker_jaas.conf @@ -0,0 +1,15 @@ +KafkaServer { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/kafka/secrets/kerberized_kafka.keytab" + principal="kafka/kerberized_kafka1@TEST.CONFLUENT.IO"; +}; +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/kafka/secrets/zkclient.keytab" + principal="zkclient@TEST.CONFLUENT.IO"; +}; + diff --git a/tests/integration/tskk/secrets/krb.conf b/tests/integration/tskk/secrets/krb.conf new file mode 100644 index 00000000000..fcc9ba22991 --- /dev/null +++ b/tests/integration/tskk/secrets/krb.conf @@ -0,0 +1,22 @@ +[logging] + default = FILE:/var/log/kerberos/krb5libs.log + kdc = FILE:/var/log/kerberos/krb5kdc.log + admin_server = FILE:/var/log/kerberos/kadmind.log + +[libdefaults] + default_realm = TEST.CONFLUENT.IO + dns_lookup_realm = false + dns_lookup_kdc = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + +[realms] + TEST.CONFLUENT.IO = { + kdc = kafka_kerberos + admin_server = kafka_kerberos + } + +[domain_realm] + .TEST.CONFLUENT.IO = TEST.CONFLUENT.IO + TEST.CONFLUENT.IO = TEST.CONFLUENT.IO diff --git a/tests/integration/tskk/secrets/zookeeper_jaas.conf b/tests/integration/tskk/secrets/zookeeper_jaas.conf new file mode 100644 index 00000000000..c9022d902d9 --- /dev/null +++ b/tests/integration/tskk/secrets/zookeeper_jaas.conf @@ -0,0 +1,14 @@ +Server { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/kafka/secrets/kafka_kerberized_zookeeper.keytab" + principal="zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO"; +}; +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab="/etc/kafka/secrets/zkclient.keytab" + principal="zkclient@TEST.CONFLUENT.IO"; +}; diff --git a/tests/integration/tskk/test.py b/tests/integration/tskk/test.py new file mode 100644 index 00000000000..348c85c6ec7 --- /dev/null +++ b/tests/integration/tskk/test.py @@ -0,0 +1,137 @@ +import os.path as p +import random +import threading +import time +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV +from helpers.client import QueryRuntimeException +from helpers.network import PartitionManager + +import json +import subprocess +import kafka.errors +from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection +from kafka.admin import NewTopic +from kafka.protocol.admin import DescribeGroupsResponse_v1, DescribeGroupsRequest_v1 +from kafka.protocol.group import MemberAssignment +import socket +from google.protobuf.internal.encoder import _VarintBytes + +""" +protoc --version +libprotoc 3.0.0 + +# to create kafka_pb2.py +protoc --python_out=. kafka.proto +""" +import kafka_pb2 + + +# TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. +# TODO: add test for SELECT LIMIT is working. + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance', + config_dir='configs', + main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], + with_kerberized_kafka=True, + # with_zookeeper=True, + clickhouse_path_dir='clickhouse_path') +kafka_id = '' # instance.cluster.kafka_docker_id + + +# Helpers + +def check_kafka_is_available(): + + # credentials are needed + + p = subprocess.Popen(('docker', + 'exec', + '-i', + kafka_id, + '/usr/bin/kafka-broker-api-versions', + '--bootstrap-server', + 'localhost:9093'), + stdout=subprocess.PIPE) + p.communicate() + return p.returncode == 0 + + +def wait_kafka_is_available(max_retries=50): + retries = 0 + while True: + if check_kafka_is_available(): + break + else: + retries += 1 + if retries > max_retries: + raise "Kafka is not available" + print("Waiting for Kafka to start up") + time.sleep(1) + + +def kafka_produce(topic, messages, timestamp=None): + producer = KafkaProducer(bootstrap_servers="localhost:9093") + for message in messages: + producer.send(topic=topic, value=message, timestamp_ms=timestamp) + producer.flush() + print ("Produced {} messages for topic {}".format(len(messages), topic)) + + + +# Fixtures + +@pytest.fixture(scope="module") +def kafka_cluster(): + try: + global kafka_id + cluster.start() + kafka_id = instance.cluster.kerberized_kafka_docker_id + print("kafka_id is {}".format(kafka_id)) + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def kafka_setup_teardown(): + instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') + wait_kafka_is_available() + print("kafka is available - running test") + time.sleep(60) + yield # run test + +# Tests + +@pytest.mark.timeout(1000) # wait to build containers +def test_kafka_json_as_string(kafka_cluster): + kafka_produce('kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + + instance.query(''' + CREATE TABLE test.kafka (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string', + kafka_group_name = 'kafka_json_as_string', + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + ''') + + result = instance.query('SELECT * FROM test.kafka;') + expected = '''\ +{"t": 123, "e": {"x": "woof"} } +{"t": 124, "e": {"x": "test"} } +{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} +''' + assert TSV(result) == TSV(expected) + assert instance.contains_in_log("Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") + + +if __name__ == '__main__': + cluster.start() + raw_input("Cluster created, press any key to destroy...") + cluster.shutdown() diff --git a/tests/integration/tskk/test_kafka_json.reference b/tests/integration/tskk/test_kafka_json.reference new file mode 100644 index 00000000000..959bb2aad74 --- /dev/null +++ b/tests/integration/tskk/test_kafka_json.reference @@ -0,0 +1,50 @@ +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +20 20 +21 21 +22 22 +23 23 +24 24 +25 25 +26 26 +27 27 +28 28 +29 29 +30 30 +31 31 +32 32 +33 33 +34 34 +35 35 +36 36 +37 37 +38 38 +39 39 +40 40 +41 41 +42 42 +43 43 +44 44 +45 45 +46 46 +47 47 +48 48 +49 49 diff --git a/tests/integration/tskk/test_kafka_virtual1.reference b/tests/integration/tskk/test_kafka_virtual1.reference new file mode 100644 index 00000000000..30c2f6e1c59 --- /dev/null +++ b/tests/integration/tskk/test_kafka_virtual1.reference @@ -0,0 +1,50 @@ + 0 virt1 0 0 0 0000-00-00 00:00:00 + 1 virt1 1 0 0 0000-00-00 00:00:00 + 2 virt1 2 0 0 0000-00-00 00:00:00 + 3 virt1 3 0 0 0000-00-00 00:00:00 + 4 virt1 4 0 0 0000-00-00 00:00:00 + 5 virt1 5 0 0 0000-00-00 00:00:00 + 6 virt1 6 0 0 0000-00-00 00:00:00 + 7 virt1 7 0 0 0000-00-00 00:00:00 + 8 virt1 8 0 0 0000-00-00 00:00:00 + 9 virt1 9 0 0 0000-00-00 00:00:00 + 10 virt1 10 0 0 0000-00-00 00:00:00 + 11 virt1 11 0 0 0000-00-00 00:00:00 + 12 virt1 12 0 0 0000-00-00 00:00:00 + 13 virt1 13 0 0 0000-00-00 00:00:00 + 14 virt1 14 0 0 0000-00-00 00:00:00 + 15 virt1 15 0 0 0000-00-00 00:00:00 + 16 virt1 16 0 0 0000-00-00 00:00:00 + 17 virt1 17 0 0 0000-00-00 00:00:00 + 18 virt1 18 0 0 0000-00-00 00:00:00 + 19 virt1 19 0 0 0000-00-00 00:00:00 + 20 virt1 20 0 0 0000-00-00 00:00:00 + 21 virt1 21 0 0 0000-00-00 00:00:00 + 22 virt1 22 0 0 0000-00-00 00:00:00 + 23 virt1 23 0 0 0000-00-00 00:00:00 + 24 virt1 24 0 0 0000-00-00 00:00:00 + 25 virt1 25 1 0 0000-00-00 00:00:00 + 26 virt1 26 1 0 0000-00-00 00:00:00 + 27 virt1 27 1 0 0000-00-00 00:00:00 + 28 virt1 28 1 0 0000-00-00 00:00:00 + 29 virt1 29 1 0 0000-00-00 00:00:00 + 30 virt1 30 1 0 0000-00-00 00:00:00 + 31 virt1 31 1 0 0000-00-00 00:00:00 + 32 virt1 32 1 0 0000-00-00 00:00:00 + 33 virt1 33 1 0 0000-00-00 00:00:00 + 34 virt1 34 1 0 0000-00-00 00:00:00 + 35 virt1 35 1 0 0000-00-00 00:00:00 + 36 virt1 36 1 0 0000-00-00 00:00:00 + 37 virt1 37 1 0 0000-00-00 00:00:00 + 38 virt1 38 1 0 0000-00-00 00:00:00 + 39 virt1 39 1 0 0000-00-00 00:00:00 + 40 virt1 40 1 0 0000-00-00 00:00:00 + 41 virt1 41 1 0 0000-00-00 00:00:00 + 42 virt1 42 1 0 0000-00-00 00:00:00 + 43 virt1 43 1 0 0000-00-00 00:00:00 + 44 virt1 44 1 0 0000-00-00 00:00:00 + 45 virt1 45 1 0 0000-00-00 00:00:00 + 46 virt1 46 1 0 0000-00-00 00:00:00 + 47 virt1 47 1 0 0000-00-00 00:00:00 + 48 virt1 48 1 0 0000-00-00 00:00:00 + 49 virt1 49 1 0 0000-00-00 00:00:00 diff --git a/tests/integration/tskk/test_kafka_virtual2.reference b/tests/integration/tskk/test_kafka_virtual2.reference new file mode 100644 index 00000000000..afb9a64f4fc --- /dev/null +++ b/tests/integration/tskk/test_kafka_virtual2.reference @@ -0,0 +1,50 @@ + 0 virt2 0 0 0 0000-00-00 00:00:00 + 1 virt2 1 1 0 0000-00-00 00:00:00 + 2 virt2 2 2 0 0000-00-00 00:00:00 + 3 virt2 3 3 0 0000-00-00 00:00:00 + 4 virt2 4 4 0 0000-00-00 00:00:00 + 5 virt2 5 5 0 0000-00-00 00:00:00 + 6 virt2 6 6 0 0000-00-00 00:00:00 + 7 virt2 7 7 0 0000-00-00 00:00:00 + 8 virt2 8 8 0 0000-00-00 00:00:00 + 9 virt2 9 9 0 0000-00-00 00:00:00 + 10 virt2 10 10 0 0000-00-00 00:00:00 + 11 virt2 11 11 0 0000-00-00 00:00:00 + 12 virt2 12 12 0 0000-00-00 00:00:00 + 13 virt2 13 13 0 0000-00-00 00:00:00 + 14 virt2 14 14 0 0000-00-00 00:00:00 + 15 virt2 15 15 0 0000-00-00 00:00:00 + 16 virt2 16 16 0 0000-00-00 00:00:00 + 17 virt2 17 17 0 0000-00-00 00:00:00 + 18 virt2 18 18 0 0000-00-00 00:00:00 + 19 virt2 19 19 0 0000-00-00 00:00:00 + 20 virt2 20 20 0 0000-00-00 00:00:00 + 21 virt2 21 21 0 0000-00-00 00:00:00 + 22 virt2 22 22 0 0000-00-00 00:00:00 + 23 virt2 23 23 0 0000-00-00 00:00:00 + 24 virt2 24 24 0 0000-00-00 00:00:00 + 25 virt2 25 25 0 0000-00-00 00:00:00 + 26 virt2 26 26 0 0000-00-00 00:00:00 + 27 virt2 27 27 0 0000-00-00 00:00:00 + 28 virt2 28 28 0 0000-00-00 00:00:00 + 29 virt2 29 29 0 0000-00-00 00:00:00 + 30 virt2 30 30 0 0000-00-00 00:00:00 + 31 virt2 31 31 0 0000-00-00 00:00:00 + 32 virt2 32 32 0 0000-00-00 00:00:00 + 33 virt2 33 33 0 0000-00-00 00:00:00 + 34 virt2 34 34 0 0000-00-00 00:00:00 + 35 virt2 35 35 0 0000-00-00 00:00:00 + 36 virt2 36 36 0 0000-00-00 00:00:00 + 37 virt2 37 37 0 0000-00-00 00:00:00 + 38 virt2 38 38 0 0000-00-00 00:00:00 + 39 virt2 39 39 0 0000-00-00 00:00:00 + 40 virt2 40 40 0 0000-00-00 00:00:00 + 41 virt2 41 41 0 0000-00-00 00:00:00 + 42 virt2 42 42 0 0000-00-00 00:00:00 + 43 virt2 43 43 0 0000-00-00 00:00:00 + 44 virt2 44 44 0 0000-00-00 00:00:00 + 45 virt2 45 45 0 0000-00-00 00:00:00 + 46 virt2 46 46 0 0000-00-00 00:00:00 + 47 virt2 47 47 0 0000-00-00 00:00:00 + 48 virt2 48 48 0 0000-00-00 00:00:00 + 49 virt2 49 49 0 0000-00-00 00:00:00 From 2d5c31d5883e1ebb6bfc5f5da1266f89929574c1 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 25 Aug 2020 10:44:31 +0300 Subject: [PATCH 105/390] test_storage_kerberized_kafka cleanup + negative test --- .../table-engines/integrations/kafka.md | 16 ++++++ tests/integration/helpers/cluster.py | 9 ++-- .../test_storage_kerberized_kafka/Dockerfile | 13 +++++ .../__init__.py | 0 .../configs/kafka.xml | 2 +- .../configs/log_conf.xml | 0 .../configs/users.xml | 0 .../kafka_pb2.py | 0 .../kerberos_image_config.sh | 29 +++++------ .../secrets/broker_jaas.conf | 5 +- .../secrets/krb.conf | 8 +-- .../secrets/zookeeper_jaas.conf | 4 +- .../test.py | 48 ++++++++++-------- tests/integration/tskk/Dockerfile | 11 ---- .../format_schemas/kafka.proto | 6 --- .../format_schemas/template_row.format | 1 - .../clickhouse_path/format_schemas/test.avsc | 11 ---- .../clickhouse_path/format_schemas/test.capnp | 10 ---- .../clickhouse_path/format_schemas/test.proto | 9 ---- .../tskk/test_kafka_json.reference | 50 ------------------- .../tskk/test_kafka_virtual1.reference | 50 ------------------- .../tskk/test_kafka_virtual2.reference | 50 ------------------- 22 files changed, 81 insertions(+), 251 deletions(-) create mode 100644 tests/integration/test_storage_kerberized_kafka/Dockerfile rename tests/integration/{tskk => test_storage_kerberized_kafka}/__init__.py (100%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/configs/kafka.xml (90%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/configs/log_conf.xml (100%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/configs/users.xml (100%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/kafka_pb2.py (100%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/kerberos_image_config.sh (76%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/secrets/broker_jaas.conf (74%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/secrets/krb.conf (70%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/secrets/zookeeper_jaas.conf (72%) rename tests/integration/{tskk => test_storage_kerberized_kafka}/test.py (77%) delete mode 100644 tests/integration/tskk/Dockerfile delete mode 100755 tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto delete mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/template_row.format delete mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.avsc delete mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.capnp delete mode 100644 tests/integration/tskk/clickhouse_path/format_schemas/test.proto delete mode 100644 tests/integration/tskk/test_kafka_json.reference delete mode 100644 tests/integration/tskk/test_kafka_virtual1.reference delete mode 100644 tests/integration/tskk/test_kafka_virtual2.reference diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index fe9aa2ca25e..d0a4bc928a7 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -165,6 +165,22 @@ Similar to GraphiteMergeTree, the Kafka engine supports extended configuration u For a list of possible configuration options, see the [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). Use the underscore (`_`) instead of a dot in the ClickHouse configuration. For example, `check.crcs=true` will be `true`. +### Kerberos support {#kafka-kerberos-support} + +To deal with Kerberos-aware Kafka, add `security_protocol` child element with `sasl_plaintext` value. It is enough if Kerberos ticket-granting ticket is obtained and cached by OS facilities. +ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` and `sasl.kerberos.kinit.cmd` child elements. + +Example: + +``` xml + + + SASL_PLAINTEXT + /home/kafkauser/kafkauser.keytab + kafkauser/kafkahost@EXAMPLE.COM + +``` + ## Virtual Columns {#virtual-columns} - `_topic` — Kafka topic. diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index ab038bc702f..d836fb57cf3 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -619,10 +619,10 @@ class ClickHouseCluster: self.wait_schema_registry_to_start(120) if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: - env = os.environ.copy() - self.kerberized_kafka_instance_path = instance.path - env['KERBERIZED_KAFKA_DIR'] = self.kerberized_kafka_instance_path + '/' - subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=env) + env_var = {} + env_var['KERBERIZED_KAFKA_DIR'] = instance.path + '/' + _create_env_file(self.base_dir, env_var, ".env") + subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes']) self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') if self.with_rabbitmq and self.base_rabbitmq_cmd: @@ -1211,7 +1211,6 @@ class ClickHouseInstance: shutil.copy(self.zookeeper_config_path, conf_d_dir) if self.with_kerberized_kafka: - # shutil.copytree(p.abspath(p.join(self.base_dir, 'secrets')), p.abspath(p.join(self.path, 'secrets'))) secrets_dir = p.abspath(p.join(self.custom_config_dir, os.pardir, 'secrets')) distutils.dir_util.copy_tree(secrets_dir, p.abspath(p.join(self.path, 'secrets'))) diff --git a/tests/integration/test_storage_kerberized_kafka/Dockerfile b/tests/integration/test_storage_kerberized_kafka/Dockerfile new file mode 100644 index 00000000000..2497fed7dd7 --- /dev/null +++ b/tests/integration/test_storage_kerberized_kafka/Dockerfile @@ -0,0 +1,13 @@ +FROM centos:6.6 +# old OS to make is faster and smaller + +RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation + +EXPOSE 88 749 + +RUN touch /config.sh +# should be overwritten e.g. via docker_compose volumes +# volumes: /some_path/my_kerberos_config.sh:/config.sh:ro + + +ENTRYPOINT ["/bin/bash", "/config.sh"] diff --git a/tests/integration/tskk/__init__.py b/tests/integration/test_storage_kerberized_kafka/__init__.py similarity index 100% rename from tests/integration/tskk/__init__.py rename to tests/integration/test_storage_kerberized_kafka/__init__.py diff --git a/tests/integration/tskk/configs/kafka.xml b/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml similarity index 90% rename from tests/integration/tskk/configs/kafka.xml rename to tests/integration/test_storage_kerberized_kafka/configs/kafka.xml index f44ec0782aa..0302bd78e3f 100644 --- a/tests/integration/tskk/configs/kafka.xml +++ b/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml @@ -12,7 +12,7 @@ GSSAPI kafka /tmp/keytab/clickhouse.keytab - kafkauser/instance@TEST.CONFLUENT.IO + kafkauser/instance@TEST.CLICKHOUSE.TECH security false diff --git a/tests/integration/tskk/configs/log_conf.xml b/tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml similarity index 100% rename from tests/integration/tskk/configs/log_conf.xml rename to tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml diff --git a/tests/integration/tskk/configs/users.xml b/tests/integration/test_storage_kerberized_kafka/configs/users.xml similarity index 100% rename from tests/integration/tskk/configs/users.xml rename to tests/integration/test_storage_kerberized_kafka/configs/users.xml diff --git a/tests/integration/tskk/kafka_pb2.py b/tests/integration/test_storage_kerberized_kafka/kafka_pb2.py similarity index 100% rename from tests/integration/tskk/kafka_pb2.py rename to tests/integration/test_storage_kerberized_kafka/kafka_pb2.py diff --git a/tests/integration/tskk/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh similarity index 76% rename from tests/integration/tskk/kerberos_image_config.sh rename to tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 7b7d61312b2..51061ecdac7 100644 --- a/tests/integration/tskk/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -1,11 +1,10 @@ #!/bin/bash -# based on confluent kerberos, which is not provided as image any more [[ "TRACE" ]] && set -x -: ${REALM:=TEST.CONFLUENT.IO} -: ${DOMAIN_REALM:=test.confluent.io} +: ${REALM:=TEST.CLICKHOUSE.TECH} +: ${DOMAIN_REALM:=test.clickhouse.tech} : ${KERB_MASTER_KEY:=masterkey} : ${KERB_ADMIN_USER:=admin} : ${KERB_ADMIN_PASS:=admin} @@ -92,23 +91,19 @@ create_admin_user() { create_keytabs() { - kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO" + kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@${REALM}" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@${REALM}" - kadmin.local -q "addprinc -randkey kafka/kerberized_kafka1@TEST.CONFLUENT.IO" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/kerberized_kafka1@TEST.CONFLUENT.IO" - # kadmin.local -q "addprinc -randkey kafka/localhost@TEST.CONFLUENT.IO" - # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/localhost@TEST.CONFLUENT.IO" + kadmin.local -q "addprinc -randkey kafka/kerberized_kafka1@${REALM}" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/kerberized_kafka1@${REALM}" - kadmin.local -q "addprinc -randkey zkclient@TEST.CONFLUENT.IO" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/zkclient.keytab zkclient@TEST.CONFLUENT.IO" - - # kadmin.local -q "addprinc -randkey kafkauser@TEST.CONFLUENT.IO" - # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser@TEST.CONFLUENT.IO" + kadmin.local -q "addprinc -randkey zkclient@${REALM}" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/zkclient.keytab zkclient@${REALM}" + + + kadmin.local -q "addprinc -randkey kafkauser/instance@${REALM}" + kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser/instance@${REALM}" - kadmin.local -q "addprinc -randkey kafkauser/instance@TEST.CONFLUENT.IO" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser/instance@TEST.CONFLUENT.IO" - chmod g+r /tmp/keytab/clickhouse.keytab } diff --git a/tests/integration/tskk/secrets/broker_jaas.conf b/tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf similarity index 74% rename from tests/integration/tskk/secrets/broker_jaas.conf rename to tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf index db119698d62..8a55ec2faa0 100644 --- a/tests/integration/tskk/secrets/broker_jaas.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf @@ -3,13 +3,12 @@ KafkaServer { useKeyTab=true storeKey=true keyTab="/etc/kafka/secrets/kerberized_kafka.keytab" - principal="kafka/kerberized_kafka1@TEST.CONFLUENT.IO"; + principal="kafka/kerberized_kafka1@TEST.CLICKHOUSE.TECH"; }; Client { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="/etc/kafka/secrets/zkclient.keytab" - principal="zkclient@TEST.CONFLUENT.IO"; + principal="zkclient@TEST.CLICKHOUSE.TECH"; }; - diff --git a/tests/integration/tskk/secrets/krb.conf b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf similarity index 70% rename from tests/integration/tskk/secrets/krb.conf rename to tests/integration/test_storage_kerberized_kafka/secrets/krb.conf index fcc9ba22991..b36fe5bbe98 100644 --- a/tests/integration/tskk/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf @@ -4,7 +4,7 @@ admin_server = FILE:/var/log/kerberos/kadmind.log [libdefaults] - default_realm = TEST.CONFLUENT.IO + default_realm = TEST.CLICKHOUSE.TECH dns_lookup_realm = false dns_lookup_kdc = false ticket_lifetime = 24h @@ -12,11 +12,11 @@ forwardable = true [realms] - TEST.CONFLUENT.IO = { + TEST.CLICKHOUSE.TECH = { kdc = kafka_kerberos admin_server = kafka_kerberos } [domain_realm] - .TEST.CONFLUENT.IO = TEST.CONFLUENT.IO - TEST.CONFLUENT.IO = TEST.CONFLUENT.IO + .TEST.CLICKHOUSE.TECH = TEST.CLICKHOUSE.TECH + TEST.CLICKHOUSE.TECH = TEST.CLICKHOUSE.TECH diff --git a/tests/integration/tskk/secrets/zookeeper_jaas.conf b/tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf similarity index 72% rename from tests/integration/tskk/secrets/zookeeper_jaas.conf rename to tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf index c9022d902d9..1b1f8103f42 100644 --- a/tests/integration/tskk/secrets/zookeeper_jaas.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf @@ -3,12 +3,12 @@ Server { useKeyTab=true storeKey=true keyTab="/etc/kafka/secrets/kafka_kerberized_zookeeper.keytab" - principal="zookeeper/kafka_kerberized_zookeeper@TEST.CONFLUENT.IO"; + principal="zookeeper/kafka_kerberized_zookeeper@TEST.CLICKHOUSE.TECH"; }; Client { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="/etc/kafka/secrets/zkclient.keytab" - principal="zkclient@TEST.CONFLUENT.IO"; + principal="zkclient@TEST.CLICKHOUSE.TECH"; }; diff --git a/tests/integration/tskk/test.py b/tests/integration/test_storage_kerberized_kafka/test.py similarity index 77% rename from tests/integration/tskk/test.py rename to tests/integration/test_storage_kerberized_kafka/test.py index 348c85c6ec7..a0d147cffcc 100644 --- a/tests/integration/tskk/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -17,37 +17,20 @@ from kafka.admin import NewTopic from kafka.protocol.admin import DescribeGroupsResponse_v1, DescribeGroupsRequest_v1 from kafka.protocol.group import MemberAssignment import socket -from google.protobuf.internal.encoder import _VarintBytes - -""" -protoc --version -libprotoc 3.0.0 - -# to create kafka_pb2.py -protoc --python_out=. kafka.proto -""" -import kafka_pb2 - - -# TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. -# TODO: add test for SELECT LIMIT is working. cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', config_dir='configs', main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], with_kerberized_kafka=True, - # with_zookeeper=True, - clickhouse_path_dir='clickhouse_path') + ) kafka_id = '' # instance.cluster.kafka_docker_id - # Helpers def check_kafka_is_available(): - # credentials are needed - + # plaintext p = subprocess.Popen(('docker', 'exec', '-i', @@ -102,12 +85,11 @@ def kafka_setup_teardown(): instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') wait_kafka_is_available() print("kafka is available - running test") - time.sleep(60) yield # run test # Tests -@pytest.mark.timeout(1000) # wait to build containers +@pytest.mark.timeout(180) # wait to build containers def test_kafka_json_as_string(kafka_cluster): kafka_produce('kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) @@ -130,6 +112,30 @@ def test_kafka_json_as_string(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") +def test_kafka_json_as_string_no_kdc(kafka_cluster): + kafka_produce('kafka_json_as_string_no_kdc', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + + kafka_cluster.pause_container('kafka_kerberos') + + instance.query(''' + CREATE TABLE test.kafka_no_kdc (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string_no_kdc', + kafka_group_name = 'kafka_json_as_string_no_kdc', + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + ''') + + result = instance.query('SELECT * FROM test.kafka_no_kdc;') + expected = '' + + kafka_cluster.unpause_container('kafka_kerberos') + + + assert TSV(result) == TSV(expected) + assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") + if __name__ == '__main__': cluster.start() diff --git a/tests/integration/tskk/Dockerfile b/tests/integration/tskk/Dockerfile deleted file mode 100644 index 2459b6980ce..00000000000 --- a/tests/integration/tskk/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM centos:6.6 - -# based on confluent kerberos, which is not provided as image any more - -RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation - -EXPOSE 88 749 - -ADD ./kerberos_image_config.sh /config.sh - -ENTRYPOINT ["/bin/bash", "/config.sh"] diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto b/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto deleted file mode 100755 index 96b24be4938..00000000000 --- a/tests/integration/tskk/clickhouse_path/format_schemas/kafka.proto +++ /dev/null @@ -1,6 +0,0 @@ -syntax = "proto3"; - -message KeyValuePair { - uint64 key = 1; - string value = 2; -} \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format b/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format deleted file mode 100644 index c910aa8589e..00000000000 --- a/tests/integration/tskk/clickhouse_path/format_schemas/template_row.format +++ /dev/null @@ -1 +0,0 @@ -(id = ${id:Escaped}, blockNo = ${blockNo:Escaped}, val1 = ${val1:CSV}, val2 = ${val2:Escaped}, val3 = ${val3:Escaped}) \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc b/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc deleted file mode 100644 index caf693313de..00000000000 --- a/tests/integration/tskk/clickhouse_path/format_schemas/test.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "record", - "name": "row", - "fields": [ - {"name": "id", "type": "long"}, - {"name": "blockNo", "type": "int"}, - {"name": "val1", "type": "string"}, - {"name": "val2", "type": "float"}, - {"name": "val3", "type": "int"} - ] - } \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp b/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp deleted file mode 100644 index 44f1961205b..00000000000 --- a/tests/integration/tskk/clickhouse_path/format_schemas/test.capnp +++ /dev/null @@ -1,10 +0,0 @@ -@0x99f75f775fe63dae; - -struct TestRecordStruct -{ - id @0 : Int64; - blockNo @1 : UInt16; - val1 @2 : Text; - val2 @3 : Float32; - val3 @4 : UInt8; -} \ No newline at end of file diff --git a/tests/integration/tskk/clickhouse_path/format_schemas/test.proto b/tests/integration/tskk/clickhouse_path/format_schemas/test.proto deleted file mode 100644 index cabdff04a78..00000000000 --- a/tests/integration/tskk/clickhouse_path/format_schemas/test.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; - -message TestMessage { - int64 id = 1; - uint32 blockNo = 2; - string val1 = 3; - float val2 = 4; - uint32 val3 = 5; -}; diff --git a/tests/integration/tskk/test_kafka_json.reference b/tests/integration/tskk/test_kafka_json.reference deleted file mode 100644 index 959bb2aad74..00000000000 --- a/tests/integration/tskk/test_kafka_json.reference +++ /dev/null @@ -1,50 +0,0 @@ -0 0 -1 1 -2 2 -3 3 -4 4 -5 5 -6 6 -7 7 -8 8 -9 9 -10 10 -11 11 -12 12 -13 13 -14 14 -15 15 -16 16 -17 17 -18 18 -19 19 -20 20 -21 21 -22 22 -23 23 -24 24 -25 25 -26 26 -27 27 -28 28 -29 29 -30 30 -31 31 -32 32 -33 33 -34 34 -35 35 -36 36 -37 37 -38 38 -39 39 -40 40 -41 41 -42 42 -43 43 -44 44 -45 45 -46 46 -47 47 -48 48 -49 49 diff --git a/tests/integration/tskk/test_kafka_virtual1.reference b/tests/integration/tskk/test_kafka_virtual1.reference deleted file mode 100644 index 30c2f6e1c59..00000000000 --- a/tests/integration/tskk/test_kafka_virtual1.reference +++ /dev/null @@ -1,50 +0,0 @@ - 0 virt1 0 0 0 0000-00-00 00:00:00 - 1 virt1 1 0 0 0000-00-00 00:00:00 - 2 virt1 2 0 0 0000-00-00 00:00:00 - 3 virt1 3 0 0 0000-00-00 00:00:00 - 4 virt1 4 0 0 0000-00-00 00:00:00 - 5 virt1 5 0 0 0000-00-00 00:00:00 - 6 virt1 6 0 0 0000-00-00 00:00:00 - 7 virt1 7 0 0 0000-00-00 00:00:00 - 8 virt1 8 0 0 0000-00-00 00:00:00 - 9 virt1 9 0 0 0000-00-00 00:00:00 - 10 virt1 10 0 0 0000-00-00 00:00:00 - 11 virt1 11 0 0 0000-00-00 00:00:00 - 12 virt1 12 0 0 0000-00-00 00:00:00 - 13 virt1 13 0 0 0000-00-00 00:00:00 - 14 virt1 14 0 0 0000-00-00 00:00:00 - 15 virt1 15 0 0 0000-00-00 00:00:00 - 16 virt1 16 0 0 0000-00-00 00:00:00 - 17 virt1 17 0 0 0000-00-00 00:00:00 - 18 virt1 18 0 0 0000-00-00 00:00:00 - 19 virt1 19 0 0 0000-00-00 00:00:00 - 20 virt1 20 0 0 0000-00-00 00:00:00 - 21 virt1 21 0 0 0000-00-00 00:00:00 - 22 virt1 22 0 0 0000-00-00 00:00:00 - 23 virt1 23 0 0 0000-00-00 00:00:00 - 24 virt1 24 0 0 0000-00-00 00:00:00 - 25 virt1 25 1 0 0000-00-00 00:00:00 - 26 virt1 26 1 0 0000-00-00 00:00:00 - 27 virt1 27 1 0 0000-00-00 00:00:00 - 28 virt1 28 1 0 0000-00-00 00:00:00 - 29 virt1 29 1 0 0000-00-00 00:00:00 - 30 virt1 30 1 0 0000-00-00 00:00:00 - 31 virt1 31 1 0 0000-00-00 00:00:00 - 32 virt1 32 1 0 0000-00-00 00:00:00 - 33 virt1 33 1 0 0000-00-00 00:00:00 - 34 virt1 34 1 0 0000-00-00 00:00:00 - 35 virt1 35 1 0 0000-00-00 00:00:00 - 36 virt1 36 1 0 0000-00-00 00:00:00 - 37 virt1 37 1 0 0000-00-00 00:00:00 - 38 virt1 38 1 0 0000-00-00 00:00:00 - 39 virt1 39 1 0 0000-00-00 00:00:00 - 40 virt1 40 1 0 0000-00-00 00:00:00 - 41 virt1 41 1 0 0000-00-00 00:00:00 - 42 virt1 42 1 0 0000-00-00 00:00:00 - 43 virt1 43 1 0 0000-00-00 00:00:00 - 44 virt1 44 1 0 0000-00-00 00:00:00 - 45 virt1 45 1 0 0000-00-00 00:00:00 - 46 virt1 46 1 0 0000-00-00 00:00:00 - 47 virt1 47 1 0 0000-00-00 00:00:00 - 48 virt1 48 1 0 0000-00-00 00:00:00 - 49 virt1 49 1 0 0000-00-00 00:00:00 diff --git a/tests/integration/tskk/test_kafka_virtual2.reference b/tests/integration/tskk/test_kafka_virtual2.reference deleted file mode 100644 index afb9a64f4fc..00000000000 --- a/tests/integration/tskk/test_kafka_virtual2.reference +++ /dev/null @@ -1,50 +0,0 @@ - 0 virt2 0 0 0 0000-00-00 00:00:00 - 1 virt2 1 1 0 0000-00-00 00:00:00 - 2 virt2 2 2 0 0000-00-00 00:00:00 - 3 virt2 3 3 0 0000-00-00 00:00:00 - 4 virt2 4 4 0 0000-00-00 00:00:00 - 5 virt2 5 5 0 0000-00-00 00:00:00 - 6 virt2 6 6 0 0000-00-00 00:00:00 - 7 virt2 7 7 0 0000-00-00 00:00:00 - 8 virt2 8 8 0 0000-00-00 00:00:00 - 9 virt2 9 9 0 0000-00-00 00:00:00 - 10 virt2 10 10 0 0000-00-00 00:00:00 - 11 virt2 11 11 0 0000-00-00 00:00:00 - 12 virt2 12 12 0 0000-00-00 00:00:00 - 13 virt2 13 13 0 0000-00-00 00:00:00 - 14 virt2 14 14 0 0000-00-00 00:00:00 - 15 virt2 15 15 0 0000-00-00 00:00:00 - 16 virt2 16 16 0 0000-00-00 00:00:00 - 17 virt2 17 17 0 0000-00-00 00:00:00 - 18 virt2 18 18 0 0000-00-00 00:00:00 - 19 virt2 19 19 0 0000-00-00 00:00:00 - 20 virt2 20 20 0 0000-00-00 00:00:00 - 21 virt2 21 21 0 0000-00-00 00:00:00 - 22 virt2 22 22 0 0000-00-00 00:00:00 - 23 virt2 23 23 0 0000-00-00 00:00:00 - 24 virt2 24 24 0 0000-00-00 00:00:00 - 25 virt2 25 25 0 0000-00-00 00:00:00 - 26 virt2 26 26 0 0000-00-00 00:00:00 - 27 virt2 27 27 0 0000-00-00 00:00:00 - 28 virt2 28 28 0 0000-00-00 00:00:00 - 29 virt2 29 29 0 0000-00-00 00:00:00 - 30 virt2 30 30 0 0000-00-00 00:00:00 - 31 virt2 31 31 0 0000-00-00 00:00:00 - 32 virt2 32 32 0 0000-00-00 00:00:00 - 33 virt2 33 33 0 0000-00-00 00:00:00 - 34 virt2 34 34 0 0000-00-00 00:00:00 - 35 virt2 35 35 0 0000-00-00 00:00:00 - 36 virt2 36 36 0 0000-00-00 00:00:00 - 37 virt2 37 37 0 0000-00-00 00:00:00 - 38 virt2 38 38 0 0000-00-00 00:00:00 - 39 virt2 39 39 0 0000-00-00 00:00:00 - 40 virt2 40 40 0 0000-00-00 00:00:00 - 41 virt2 41 41 0 0000-00-00 00:00:00 - 42 virt2 42 42 0 0000-00-00 00:00:00 - 43 virt2 43 43 0 0000-00-00 00:00:00 - 44 virt2 44 44 0 0000-00-00 00:00:00 - 45 virt2 45 45 0 0000-00-00 00:00:00 - 46 virt2 46 46 0 0000-00-00 00:00:00 - 47 virt2 47 47 0 0000-00-00 00:00:00 - 48 virt2 48 48 0 0000-00-00 00:00:00 - 49 virt2 49 49 0 0000-00-00 00:00:00 From 7a6f7a872a5576270e4069f46dbb865bda0900fc Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 25 Aug 2020 11:46:03 +0300 Subject: [PATCH 106/390] forgotten docker changes --- docker/test/integration/base/Dockerfile | 3 ++- .../runner/compose/docker_compose_kerberized_kafka.yml | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 53627c78208..88f05d9482f 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -17,7 +17,8 @@ RUN apt-get update \ odbc-postgresql \ sqlite3 \ curl \ - tar + tar \ + krb5-user RUN rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index f06e26c76cb..01f053256ff 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -42,6 +42,7 @@ services: KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true" volumes: - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/etc/kafka/secrets + - /dev/urandom:/dev/random depends_on: - kafka_kerberized_zookeeper - kafka_kerberos @@ -49,8 +50,11 @@ services: - label:disable kafka_kerberos: - build: /home/ilejn/projects/ClickHouse/tests/integration/tskk + # build: ${KERBERIZED_KAFKA_DIR:-../..--../../../../tests/integration/tskk} + image: ilejn/kdc:latest hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/tmp/keytab + - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/../../kerberos_image_config.sh:/config.sh + - /dev/urandom:/dev/random ports: [88, 749] From a2e72ec37a5c80ae0b8ed481e5928f6bb6318626 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 26 Aug 2020 23:43:02 +0300 Subject: [PATCH 107/390] test via runner is Ok, switched to custom images --- .../docker_compose_kerberized_kafka.yml | 12 ++++++------ tests/integration/helpers/cluster.py | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 01f053256ff..8ba9ea44eac 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -11,7 +11,7 @@ services: ZOOKEEPER_SERVERS: "kafka_kerberized_zookeeper:2888:3888" KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dsun.security.krb5.debug=true" volumes: - - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/etc/kafka/secrets + - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets - /dev/urandom:/dev/random depends_on: - kafka_kerberos @@ -32,7 +32,7 @@ services: # KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:9092,OUTSIDE://kerberized_kafka1:19092 KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: GSSAPI KAFKA_SASL_ENABLED_MECHANISMS: GSSAPI - KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka + KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: OUTSIDE:SASL_PLAINTEXT,UNSECURED_OUTSIDE:PLAINTEXT,UNSECURED_INSIDE:PLAINTEXT, KAFKA_INTER_BROKER_LISTENER_NAME: OUTSIDE KAFKA_BROKER_ID: 1 @@ -41,7 +41,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true" volumes: - - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/etc/kafka/secrets + - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets - /dev/urandom:/dev/random depends_on: - kafka_kerberized_zookeeper @@ -50,11 +50,11 @@ services: - label:disable kafka_kerberos: - # build: ${KERBERIZED_KAFKA_DIR:-../..--../../../../tests/integration/tskk} + # build: ${KERBERIZED_KAFKA_DIR} image: ilejn/kdc:latest hostname: kafka_kerberos volumes: - - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/secrets:/tmp/keytab - - ${KERBERIZED_KAFKA_DIR:-../../../../../tests/integration/tskk/_instances/instance}/../../kerberos_image_config.sh:/config.sh + - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab + - ${KERBERIZED_KAFKA_DIR}/../../kerberos_image_config.sh:/config.sh - /dev/urandom:/dev/random ports: [88, 749] diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index d836fb57cf3..358fc9d8230 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -45,6 +45,10 @@ def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): f.write("=".join([var, value]) + "\n") return full_path +def remove_files(files): + for a_file in files: + if os.path.isfile(a_file): + os.remove(a_file) def subprocess_check_call(args): # Uncomment for debugging @@ -175,7 +179,11 @@ class ClickHouseCluster: clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, +<<<<<<< HEAD hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, +======= + hostname=None, env_variables=None, image="ilejn/yandex_clickhouse-integration-test", +>>>>>>> test via runner is Ok, switched to custom images stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None): """Add an instance to the cluster. @@ -621,10 +629,15 @@ class ClickHouseCluster: if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: env_var = {} env_var['KERBERIZED_KAFKA_DIR'] = instance.path + '/' - _create_env_file(self.base_dir, env_var, ".env") - subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes']) - self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') + # different docker_compose versions look for .env in different places + # -- env-file too recent to rely on it + files_to_cleanup = [] + files_to_cleanup.append(_create_env_file(self.base_dir, env_var, ".env")) + files_to_cleanup.append(_create_env_file(os.getcwd(), env_var, ".env")) + subprocess_check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes']) + self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') + remove_files(files_to_cleanup) if self.with_rabbitmq and self.base_rabbitmq_cmd: subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) self.rabbitmq_docker_id = self.get_instance_docker_id('rabbitmq1') From c227f91636ee4dc71726806ed79663578b1d0f2e Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 31 Aug 2020 11:50:04 +0300 Subject: [PATCH 108/390] remove_files improved, proper docker images, tiny cleanup --- .../docker_compose_kerberized_kafka.yml | 2 +- tests/integration/helpers/cluster.py | 8 +- .../kafka_pb2.py | 76 ------------------- .../kerberos_image_config.sh | 6 +- 4 files changed, 7 insertions(+), 85 deletions(-) delete mode 100644 tests/integration/test_storage_kerberized_kafka/kafka_pb2.py diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 8ba9ea44eac..fa41e1ee6a5 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -51,7 +51,7 @@ services: kafka_kerberos: # build: ${KERBERIZED_KAFKA_DIR} - image: ilejn/kdc:latest + image: arenadata/kdc:latest hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 358fc9d8230..99b8aa38766 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -47,8 +47,10 @@ def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): def remove_files(files): for a_file in files: - if os.path.isfile(a_file): + try: os.remove(a_file) + except: + pass def subprocess_check_call(args): # Uncomment for debugging @@ -179,11 +181,7 @@ class ClickHouseCluster: clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, -<<<<<<< HEAD hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, -======= - hostname=None, env_variables=None, image="ilejn/yandex_clickhouse-integration-test", ->>>>>>> test via runner is Ok, switched to custom images stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None): """Add an instance to the cluster. diff --git a/tests/integration/test_storage_kerberized_kafka/kafka_pb2.py b/tests/integration/test_storage_kerberized_kafka/kafka_pb2.py deleted file mode 100644 index 79890682125..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/kafka_pb2.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: clickhouse_path/format_schemas/kafka.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/kafka.proto', - package='', - syntax='proto3', - serialized_pb=_b('\n*clickhouse_path/format_schemas/kafka.proto\"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_KEYVALUEPAIR = _descriptor.Descriptor( - name='KeyValuePair', - full_name='KeyValuePair', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='KeyValuePair.key', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='KeyValuePair.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=46, - serialized_end=88, -) - -DESCRIPTOR.message_types_by_name['KeyValuePair'] = _KEYVALUEPAIR - -KeyValuePair = _reflection.GeneratedProtocolMessageType('KeyValuePair', (_message.Message,), dict( - DESCRIPTOR = _KEYVALUEPAIR, - __module__ = 'clickhouse_path.format_schemas.kafka_pb2' - # @@protoc_insertion_point(class_scope:KeyValuePair) - )) -_sym_db.RegisterMessage(KeyValuePair) - - -# @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 51061ecdac7..5a9669e3db5 100644 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -97,14 +97,14 @@ create_keytabs() { kadmin.local -q "addprinc -randkey kafka/kerberized_kafka1@${REALM}" kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/kerberized_kafka1@${REALM}" - kadmin.local -q "addprinc -randkey zkclient@${REALM}" + kadmin.local -q "addprinc -randkey zkclient@${REALM}" kadmin.local -q "ktadd -norandkey -k /tmp/keytab/zkclient.keytab zkclient@${REALM}" - kadmin.local -q "addprinc -randkey kafkauser/instance@${REALM}" + kadmin.local -q "addprinc -randkey kafkauser/instance@${REALM}" kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser/instance@${REALM}" - chmod g+r /tmp/keytab/clickhouse.keytab + chmod g+r /tmp/keytab/clickhouse.keytab } From 1808a1e36ca974575972a11f028ad29504c09f8d Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 2 Sep 2020 11:39:58 +0000 Subject: [PATCH 109/390] short krb ticket lifetime, Dockerile renamed --- .../{Dockerfile => Dockerfile_kdc} | 0 .../test_storage_kerberized_kafka/kerberos_image_config.sh | 4 ++-- .../test_storage_kerberized_kafka/secrets/krb.conf | 4 ++-- tests/integration/test_storage_kerberized_kafka/test.py | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) rename tests/integration/test_storage_kerberized_kafka/{Dockerfile => Dockerfile_kdc} (100%) diff --git a/tests/integration/test_storage_kerberized_kafka/Dockerfile b/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc similarity index 100% rename from tests/integration/test_storage_kerberized_kafka/Dockerfile rename to tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 5a9669e3db5..419c77e12e5 100644 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -22,8 +22,8 @@ create_config() { default_realm = $REALM dns_lookup_realm = false dns_lookup_kdc = false - ticket_lifetime = 24h - renew_lifetime = 7d + ticket_lifetime = 15s + renew_lifetime = 15s forwardable = true # WARNING: We use weaker key types to simplify testing as stronger key types # require the enhanced security JCE policy file to be installed. You should diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf index b36fe5bbe98..1efdf510f22 100644 --- a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf @@ -7,8 +7,8 @@ default_realm = TEST.CLICKHOUSE.TECH dns_lookup_realm = false dns_lookup_kdc = false - ticket_lifetime = 24h - renew_lifetime = 7d + ticket_lifetime = 15s + renew_lifetime = 15s forwardable = true [realms] diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index a0d147cffcc..92b4da88f70 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -116,6 +116,7 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): kafka_produce('kafka_json_as_string_no_kdc', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) kafka_cluster.pause_container('kafka_kerberos') + time.sleep(45) # wait for ticket expiration instance.query(''' CREATE TABLE test.kafka_no_kdc (field String) From 96c2a7c77ef808505d5ca8daad05314d0cc1cc4c Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 9 Sep 2020 10:01:21 +0000 Subject: [PATCH 110/390] kerberized kafka test works after merge --- tests/integration/helpers/cluster.py | 4 ++-- tests/integration/test_storage_kerberized_kafka/test.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 99b8aa38766..ecb9babe781 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -860,6 +860,7 @@ class ClickHouseInstance: self.custom_user_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_user_configs] self.custom_dictionaries_paths = [p.abspath(p.join(base_path, c)) for c in custom_dictionaries] self.clickhouse_path_dir = p.abspath(p.join(base_path, clickhouse_path_dir)) if clickhouse_path_dir else None + self.kerberos_secrets_dir = p.abspath(p.join(base_path, 'secrets')) self.macros = macros if macros is not None else {} self.with_zookeeper = with_zookeeper self.zookeeper_config_path = zookeeper_config_path @@ -1222,8 +1223,7 @@ class ClickHouseInstance: shutil.copy(self.zookeeper_config_path, conf_d_dir) if self.with_kerberized_kafka: - secrets_dir = p.abspath(p.join(self.custom_config_dir, os.pardir, 'secrets')) - distutils.dir_util.copy_tree(secrets_dir, p.abspath(p.join(self.path, 'secrets'))) + distutils.dir_util.copy_tree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) # Copy config dir if self.custom_config_dir: diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 92b4da88f70..6ee526d9822 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -20,7 +20,6 @@ import socket cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', - config_dir='configs', main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], with_kerberized_kafka=True, ) From 66a055ddde7847a8876176cfd214f295c9e07cfa Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 9 Sep 2020 10:41:25 +0000 Subject: [PATCH 111/390] switched to cyrus-sasl-2.1 release branch --- .gitmodules | 1 + contrib/cyrus-sasl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 19f93ee8270..c68498fdbab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -186,3 +186,4 @@ [submodule "contrib/cyrus-sasl"] path = contrib/cyrus-sasl url = https://github.com/cyrusimap/cyrus-sasl + branch = cyrus-sasl-2.1 diff --git a/contrib/cyrus-sasl b/contrib/cyrus-sasl index 6054630889f..9995bf9d8e1 160000 --- a/contrib/cyrus-sasl +++ b/contrib/cyrus-sasl @@ -1 +1 @@ -Subproject commit 6054630889fd1cd8d0659573d69badcee1e23a00 +Subproject commit 9995bf9d8e14f58934d9313ac64f13780d6dd3c9 From 23853cc8cfb7c74b2f374e3528788eaa6504b744 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 9 Sep 2020 23:23:48 +0300 Subject: [PATCH 112/390] fix conflict resolution mistake --- tests/integration/helpers/cluster.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index ecb9babe781..8b11896eb4e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1225,10 +1225,6 @@ class ClickHouseInstance: if self.with_kerberized_kafka: distutils.dir_util.copy_tree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) - # Copy config dir - if self.custom_config_dir: - distutils.dir_util.copy_tree(self.custom_config_dir, configs_dir) - # Copy config.d configs print "Copy custom test config files {} to {}".format(self.custom_main_config_paths, self.config_d_dir) for path in self.custom_main_config_paths: From bcdb2277bd6eca4692b2b61a6877d129d3f4a654 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 10 Sep 2020 10:33:27 +0300 Subject: [PATCH 113/390] distutils=>shutil --- tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 8b11896eb4e..1ddb7e220e5 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1223,7 +1223,7 @@ class ClickHouseInstance: shutil.copy(self.zookeeper_config_path, conf_d_dir) if self.with_kerberized_kafka: - distutils.dir_util.copy_tree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) + shutil.copytree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) # Copy config.d configs print "Copy custom test config files {} to {}".format(self.custom_main_config_paths, self.config_d_dir) From 75836acddcf37b87a61d24d4747aecbd1daf7575 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 16 Sep 2020 00:54:58 +0300 Subject: [PATCH 114/390] code review changes --- tests/integration/helpers/cluster.py | 20 +++------------ .../Dockerfile_kdc | 2 ++ .../configs/users.xml | 25 ------------------- .../kerberos_image_config.sh | 16 ++++++------ .../test_storage_kerberized_kafka/test.py | 3 +++ 5 files changed, 16 insertions(+), 50 deletions(-) delete mode 100644 tests/integration/test_storage_kerberized_kafka/configs/users.xml diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 1ddb7e220e5..820b102e83c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -45,13 +45,6 @@ def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): f.write("=".join([var, value]) + "\n") return full_path -def remove_files(files): - for a_file in files: - try: - os.remove(a_file) - except: - pass - def subprocess_check_call(args): # Uncomment for debugging # print('run:', ' ' . join(args)) @@ -625,17 +618,10 @@ class ClickHouseCluster: self.wait_schema_registry_to_start(120) if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: - env_var = {} - env_var['KERBERIZED_KAFKA_DIR'] = instance.path + '/' - - # different docker_compose versions look for .env in different places - # -- env-file too recent to rely on it - files_to_cleanup = [] - files_to_cleanup.append(_create_env_file(self.base_dir, env_var, ".env")) - files_to_cleanup.append(_create_env_file(os.getcwd(), env_var, ".env")) - subprocess_check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes']) + env = os.environ.copy() + env['KERBERIZED_KAFKA_DIR'] = instance.path + '/' + subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=env) self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') - remove_files(files_to_cleanup) if self.with_rabbitmq and self.base_rabbitmq_cmd: subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) self.rabbitmq_docker_id = self.get_instance_docker_id('rabbitmq1') diff --git a/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc b/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc index 2497fed7dd7..3ead8e4fe85 100644 --- a/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc +++ b/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc @@ -1,3 +1,5 @@ +# docker build -t arenadata/kdc . + FROM centos:6.6 # old OS to make is faster and smaller diff --git a/tests/integration/test_storage_kerberized_kafka/configs/users.xml b/tests/integration/test_storage_kerberized_kafka/configs/users.xml deleted file mode 100644 index 246e6b069ef..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/configs/users.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - ::/0 - - default - default - - - - - - - - diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 419c77e12e5..dda10d47d94 100644 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -1,16 +1,16 @@ #!/bin/bash -[[ "TRACE" ]] && set -x +set -x # trace -: ${REALM:=TEST.CLICKHOUSE.TECH} -: ${DOMAIN_REALM:=test.clickhouse.tech} -: ${KERB_MASTER_KEY:=masterkey} -: ${KERB_ADMIN_USER:=admin} -: ${KERB_ADMIN_PASS:=admin} +: "${REALM:=TEST.CLICKHOUSE.TECH}" +: "${DOMAIN_REALM:=test.clickhouse.tech}" +: "${KERB_MASTER_KEY:=masterkey}" +: "${KERB_ADMIN_USER:=admin}" +: "${KERB_ADMIN_PASS:=admin}" create_config() { - : ${KDC_ADDRESS:=$(hostname -f)} + : "${KDC_ADDRESS:=$(hostname -f)}" cat>/etc/krb5.conf< Date: Wed, 16 Sep 2020 22:52:04 +0300 Subject: [PATCH 115/390] forgotten empty clickhouse_path dir --- .../test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR diff --git a/tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR b/tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR new file mode 100644 index 00000000000..e69de29bb2d From 624a4c4d55f64b67cacbab9fed4e30bc55c2d2cb Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 17 Sep 2020 16:50:52 +0300 Subject: [PATCH 116/390] kdc Dockerfile integrated --- docker/images.json | 4 ++++ .../test/integration/kerberos_kdc/Dockerfile | 2 +- .../runner/compose/docker_compose_kerberized_kafka.yml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) rename tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc => docker/test/integration/kerberos_kdc/Dockerfile (86%) diff --git a/docker/images.json b/docker/images.json index 8c2cb35b004..e9e91864e1e 100644 --- a/docker/images.json +++ b/docker/images.json @@ -133,6 +133,10 @@ "name": "yandex/clickhouse-postgresql-java-client", "dependent": [] }, + "docker/test/integration/kerberos_kdc": { + "name": "yandex/clickhouse-kerberos-kdc", + "dependent": [] + }, "docker/test/base": { "name": "yandex/clickhouse-test-base", "dependent": [ diff --git a/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc b/docker/test/integration/kerberos_kdc/Dockerfile similarity index 86% rename from tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc rename to docker/test/integration/kerberos_kdc/Dockerfile index 3ead8e4fe85..ea231b1191d 100644 --- a/tests/integration/test_storage_kerberized_kafka/Dockerfile_kdc +++ b/docker/test/integration/kerberos_kdc/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t arenadata/kdc . +# docker build -t yandex/clickhouse-kerberos-kdc . FROM centos:6.6 # old OS to make is faster and smaller diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index fa41e1ee6a5..34c453370e9 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -51,7 +51,7 @@ services: kafka_kerberos: # build: ${KERBERIZED_KAFKA_DIR} - image: arenadata/kdc:latest + image: yandex/clickhouse-kerberos-kdc:latest hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab From fb64cf210a495db124ac428712e126080358327f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 15:15:05 +0300 Subject: [PATCH 117/390] straighten the protocol version --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopierApp.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Client/Connection.cpp | 4 +- src/Common/ClickHouseRevision.cpp | 2 +- src/Common/ClickHouseRevision.h | 2 +- src/Common/StatusFile.cpp | 2 +- src/Common/config_version.h.in | 13 +---- src/DataStreams/TemporaryFileStream.h | 3 +- src/Interpreters/Aggregator.cpp | 2 +- src/Interpreters/ClientInfo.cpp | 9 ++-- src/Interpreters/ClientInfo.h | 2 +- src/Interpreters/CrashLog.cpp | 2 +- src/Interpreters/QueryLog.cpp | 4 +- src/Interpreters/QueryThreadLog.cpp | 2 +- src/Interpreters/TextLog.cpp | 2 +- src/Interpreters/TraceLog.cpp | 2 +- .../Transforms/AggregatingTransform.cpp | 3 +- src/Server/TCPHandler.cpp | 51 +++++++++---------- src/Server/TCPHandler.h | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 7 ++- .../DistributedBlockOutputStream.cpp | 7 ++- .../System/StorageSystemProcesses.cpp | 2 +- 23 files changed, 55 insertions(+), 74 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 78801e71a6f..22455d09cf2 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -781,7 +781,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() void BaseDaemon::logRevision() const { Poco::Logger::root().information("Starting " + std::string{VERSION_FULL} - + " with revision " + std::to_string(ClickHouseRevision::get()) + + " with revision " + std::to_string(ClickHouseRevision::getVersionRevision()) + ", " + build_id_info + ", PID " + std::to_string(getpid())); } diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index ec64e118f45..08a7e50a9d7 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -105,7 +105,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::get()); + LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::getVersionRevision()); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index aa947b22593..da782302eb9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -256,7 +256,7 @@ int Server::main(const std::vector & /*args*/) #endif #endif - CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); + CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); if (ThreadFuzzer::instance().isEffective()) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index d8fe865136f..f388ffed4a3 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -165,14 +165,12 @@ void Connection::sendHello() || has_control_character(password)) throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS); - auto client_revision = ClickHouseRevision::get(); - writeVarUInt(Protocol::Client::Hello, *out); writeStringBinary((DBMS_NAME " ") + client_name, *out); writeVarUInt(DBMS_VERSION_MAJOR, *out); writeVarUInt(DBMS_VERSION_MINOR, *out); // NOTE For backward compatibility of the protocol, client cannot send its version_patch. - writeVarUInt(client_revision, *out); + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); writeStringBinary(default_database, *out); /// If interserver-secret is used, one do not need password /// (NOTE we do not check for DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET, since we cannot ignore inter-server secret if it was requested) diff --git a/src/Common/ClickHouseRevision.cpp b/src/Common/ClickHouseRevision.cpp index 0b81026adca..2c52ebb064a 100644 --- a/src/Common/ClickHouseRevision.cpp +++ b/src/Common/ClickHouseRevision.cpp @@ -6,6 +6,6 @@ namespace ClickHouseRevision { - unsigned get() { return VERSION_REVISION; } + unsigned getVersionRevision() { return VERSION_REVISION; } unsigned getVersionInteger() { return VERSION_INTEGER; } } diff --git a/src/Common/ClickHouseRevision.h b/src/Common/ClickHouseRevision.h index 1d097a5bf89..86d1e3db334 100644 --- a/src/Common/ClickHouseRevision.h +++ b/src/Common/ClickHouseRevision.h @@ -2,6 +2,6 @@ namespace ClickHouseRevision { - unsigned get(); + unsigned getVersionRevision(); unsigned getVersionInteger(); } diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index 7c6bbf814a0..b21454c9ed8 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -37,7 +37,7 @@ StatusFile::FillFunction StatusFile::write_full_info = [](WriteBuffer & out) { out << "PID: " << getpid() << "\n" << "Started at: " << LocalDateTime(time(nullptr)) << "\n" - << "Revision: " << ClickHouseRevision::get() << "\n"; + << "Revision: " << ClickHouseRevision::getVersionRevision() << "\n"; }; diff --git a/src/Common/config_version.h.in b/src/Common/config_version.h.in index c3c0c6df87b..880824f8ad0 100644 --- a/src/Common/config_version.h.in +++ b/src/Common/config_version.h.in @@ -2,18 +2,7 @@ // .h autogenerated by cmake! -#cmakedefine01 USE_DBMS_TCP_PROTOCOL_VERSION - -#if USE_DBMS_TCP_PROTOCOL_VERSION - #include "Core/Defines.h" - #ifndef VERSION_REVISION - #define VERSION_REVISION DBMS_TCP_PROTOCOL_VERSION - #endif -#else - #cmakedefine VERSION_REVISION @VERSION_REVISION@ -#endif - - +#cmakedefine VERSION_REVISION @VERSION_REVISION@ #cmakedefine VERSION_NAME "@VERSION_NAME@" #define DBMS_NAME VERSION_NAME #cmakedefine VERSION_MAJOR @VERSION_MAJOR@ diff --git a/src/DataStreams/TemporaryFileStream.h b/src/DataStreams/TemporaryFileStream.h index 6871800a540..b481cef1bb2 100644 --- a/src/DataStreams/TemporaryFileStream.h +++ b/src/DataStreams/TemporaryFileStream.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -23,7 +22,7 @@ struct TemporaryFileStream TemporaryFileStream(const std::string & path) : file_in(path) , compressed_in(file_in) - , block_in(std::make_shared(compressed_in, ClickHouseRevision::get())) + , block_in(std::make_shared(compressed_in, DBMS_TCP_PROTOCOL_VERSION)) {} TemporaryFileStream(const std::string & path, const Block & header_) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 86a33dccb53..466370a22a2 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -844,7 +844,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co const std::string & path = file->path(); WriteBufferFromFile file_buf(path); CompressedWriteBuffer compressed_buf(file_buf); - NativeBlockOutputStream block_out(compressed_buf, ClickHouseRevision::get(), getHeader(false)); + NativeBlockOutputStream block_out(compressed_buf, DBMS_TCP_PROTOCOL_VERSION, getHeader(false)); LOG_DEBUG(log, "Writing part of aggregation data into temporary file {}.", path); ProfileEvents::increment(ProfileEvents::ExternalAggregationWritePart); diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 378375dcc18..71567a424c5 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #if !defined(ARCADIA_BUILD) @@ -44,7 +43,7 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) writeBinary(client_name, out); writeVarUInt(client_version_major, out); writeVarUInt(client_version_minor, out); - writeVarUInt(client_revision, out); + writeVarUInt(client_tcp_protocol_version, out); } else if (interface == Interface::HTTP) { @@ -92,7 +91,7 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) readBinary(client_name, in); readVarUInt(client_version_major, in); readVarUInt(client_version_minor, in); - readVarUInt(client_revision, in); + readVarUInt(client_tcp_protocol_version, in); } else if (interface == Interface::HTTP) { @@ -111,7 +110,7 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) readVarUInt(client_version_patch, in); else - client_version_patch = client_revision; + client_version_patch = client_tcp_protocol_version; } } @@ -137,7 +136,7 @@ void ClientInfo::fillOSUserHostNameAndVersionInfo() client_version_major = DBMS_VERSION_MAJOR; client_version_minor = DBMS_VERSION_MINOR; client_version_patch = DBMS_VERSION_PATCH; - client_revision = ClickHouseRevision::get(); + client_tcp_protocol_version = DBMS_TCP_PROTOCOL_VERSION; } diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 99426716cb2..704f1913b89 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -69,7 +69,7 @@ public: UInt64 client_version_major = 0; UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; - unsigned client_revision = 0; + unsigned client_tcp_protocol_version = 0; /// For http HTTPMethod http_method = HTTPMethod::UNKNOWN; diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 12fd57c33dc..9d84d5a18e9 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -49,7 +49,7 @@ void CrashLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(trace); columns[i++]->insert(trace_full); columns[i++]->insert(VERSION_FULL); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); String build_id_hex; #if defined(__ELF__) && !defined(__FreeBSD__) diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 62dbc114633..75e0fae615a 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -118,7 +118,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const appendClientInfo(client_info, columns, i); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); { Array threads_array; @@ -172,7 +172,7 @@ void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableCo columns[i++]->insert(client_info.os_user); columns[i++]->insert(client_info.client_hostname); columns[i++]->insert(client_info.client_name); - columns[i++]->insert(client_info.client_revision); + columns[i++]->insert(client_info.client_tcp_protocol_version); columns[i++]->insert(client_info.client_version_major); columns[i++]->insert(client_info.client_version_minor); columns[i++]->insert(client_info.client_version_patch); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 22ad60d96b4..e5a8cf7c5cf 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -93,7 +93,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const QueryLogElement::appendClientInfo(client_info, columns, i); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); if (profile_counters) { diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index d166b24ef4f..243bf6d299a 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -62,7 +62,7 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(logger_name); columns[i++]->insert(message); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); columns[i++]->insert(source_file); columns[i++]->insert(source_line); diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index c4fa7307b1a..f7e82032f49 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -43,7 +43,7 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); columns[i++]->insert(event_time); columns[i++]->insert(timestamp_ns); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); columns[i++]->insert(static_cast(trace_type)); columns[i++]->insert(thread_id); columns[i++]->insertData(query_id.data(), query_id.size()); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 42caf4b3446..0a97cc3d4cb 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -56,7 +55,7 @@ namespace public: SourceFromNativeStream(const Block & header, const std::string & path) : ISource(header), file_in(path), compressed_in(file_in), - block_in(std::make_shared(compressed_in, ClickHouseRevision::get())) + block_in(std::make_shared(compressed_in, DBMS_TCP_PROTOCOL_VERSION)) { block_in->readPrefix(); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index d3a5ea38f3f..3e1b53a3142 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -185,7 +184,7 @@ void TCPHandler::runImpl() /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_LOGS && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); @@ -220,7 +219,7 @@ void TCPHandler::runImpl() state.need_receive_data_for_input = true; /// Send ColumnsDescription for input storage. - if (client_revision >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA && query_context->getSettingsRef().input_format_defaults_for_omitted_fields) { sendTableColumns(metadata_snapshot->getColumns()); @@ -250,7 +249,7 @@ void TCPHandler::runImpl() customizeContext(*query_context); - bool may_have_embedded_data = client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA; + bool may_have_embedded_data = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA; /// Processing Query state.io = executeQuery(state.query, *query_context, false, state.stage, may_have_embedded_data); @@ -492,7 +491,7 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) state.io.out->writePrefix(); /// Send ColumnsDescription for insertion table - if (client_revision >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) { const auto & table_id = query_context->getInsertionTable(); if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) @@ -648,7 +647,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() void TCPHandler::processTablesStatusRequest() { TablesStatusRequest request; - request.read(*in, client_revision); + request.read(*in, client_tcp_protocol_version); TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) @@ -671,13 +670,13 @@ void TCPHandler::processTablesStatusRequest() } writeVarUInt(Protocol::Server::TablesStatusResponse, *out); - response.write(*out, client_revision); + response.write(*out, client_tcp_protocol_version); } void TCPHandler::receiveUnexpectedTablesStatusRequest() { TablesStatusRequest skip_request; - skip_request.read(*in, client_revision); + skip_request.read(*in, client_tcp_protocol_version); throw NetException("Unexpected packet TablesStatusRequest received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } @@ -752,7 +751,7 @@ void TCPHandler::receiveHello() readVarUInt(client_version_major, *in); readVarUInt(client_version_minor, *in); // NOTE For backward compatibility of the protocol, client cannot send its version_patch. - readVarUInt(client_revision, *in); + readVarUInt(client_tcp_protocol_version, *in); readStringBinary(default_database, *in); readStringBinary(user, *in); readStringBinary(password, *in); @@ -763,7 +762,7 @@ void TCPHandler::receiveHello() LOG_DEBUG(log, "Connected {} version {}.{}.{}, revision: {}{}{}.", client_name, client_version_major, client_version_minor, client_version_patch, - client_revision, + client_tcp_protocol_version, (!default_database.empty() ? ", database: " + default_database : ""), (!user.empty() ? ", user: " + user : "") ); @@ -802,12 +801,12 @@ void TCPHandler::sendHello() writeStringBinary(DBMS_NAME, *out); writeVarUInt(DBMS_VERSION_MAJOR, *out); writeVarUInt(DBMS_VERSION_MINOR, *out); - writeVarUInt(ClickHouseRevision::get(), *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) writeStringBinary(DateLUT::instance().getTimeZone(), *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) writeStringBinary(server_display_name, *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(DBMS_VERSION_PATCH, *out); out->next(); } @@ -894,8 +893,8 @@ void TCPHandler::receiveQuery() /// Client info ClientInfo & client_info = query_context->getClientInfo(); - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - client_info.read(*in, client_revision); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) + client_info.read(*in, client_tcp_protocol_version); /// For better support of old clients, that does not send ClientInfo. if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) @@ -905,7 +904,7 @@ void TCPHandler::receiveQuery() client_info.client_version_major = client_version_major; client_info.client_version_minor = client_version_minor; client_info.client_version_patch = client_version_patch; - client_info.client_revision = client_revision; + client_info.client_tcp_protocol_version = client_tcp_protocol_version; } /// Set fields, that are known apriori. @@ -921,7 +920,7 @@ void TCPHandler::receiveQuery() /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. - auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS + auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS : SettingsWriteFormat::BINARY; Settings passed_settings; passed_settings.read(*in, settings_format); @@ -1011,11 +1010,11 @@ void TCPHandler::receiveUnexpectedQuery() readStringBinary(skip_string, *in); ClientInfo skip_client_info; - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - skip_client_info.read(*in, client_revision); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) + skip_client_info.read(*in, client_tcp_protocol_version); Settings skip_settings; - auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS + auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS : SettingsWriteFormat::BINARY; skip_settings.read(*in, settings_format); @@ -1094,7 +1093,7 @@ void TCPHandler::receiveUnexpectedData() auto skip_block_in = std::make_shared( *maybe_compressed_in, last_block_in.header, - client_revision); + client_tcp_protocol_version); skip_block_in->read(); throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); @@ -1121,7 +1120,7 @@ void TCPHandler::initBlockInput() state.block_in = std::make_shared( *state.maybe_compressed_in, header, - client_revision); + client_tcp_protocol_version); } } @@ -1152,7 +1151,7 @@ void TCPHandler::initBlockOutput(const Block & block) state.block_out = std::make_shared( *state.maybe_compressed_out, - client_revision, + client_tcp_protocol_version, block.cloneEmpty(), !connection_context.getSettingsRef().low_cardinality_allow_in_native_format); } @@ -1165,7 +1164,7 @@ void TCPHandler::initLogsBlockOutput(const Block & block) /// Use uncompressed stream since log blocks usually contain only one row state.logs_block_out = std::make_shared( *out, - client_revision, + client_tcp_protocol_version, block.cloneEmpty(), !connection_context.getSettingsRef().low_cardinality_allow_in_native_format); } @@ -1269,7 +1268,7 @@ void TCPHandler::sendProgress() { writeVarUInt(Protocol::Server::Progress, *out); auto increment = state.progress.fetchAndResetPiecewiseAtomically(); - increment.write(*out, client_revision); + increment.write(*out, client_tcp_protocol_version); out->next(); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 3771755892f..12149d9a66f 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -123,7 +123,7 @@ private: UInt64 client_version_major = 0; UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; - UInt64 client_revision = 0; + UInt64 client_tcp_protocol_version = 0; Context connection_context; std::optional query_context; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index dfb35f62bc4..f40ce1e06fc 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -366,7 +365,7 @@ void StorageDistributedDirectoryMonitor::readHeader( UInt64 initiator_revision; readVarUInt(initiator_revision, header_buf); - if (ClickHouseRevision::get() < initiator_revision) + if (DBMS_TCP_PROTOCOL_VERSION < initiator_revision) { LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } @@ -585,7 +584,7 @@ public: explicit DirectoryMonitorBlockInputStream(const String & file_name) : in(file_name) , decompressing_in(in) - , block_in(decompressing_in, ClickHouseRevision::get()) + , block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION) , log{&Poco::Logger::get("DirectoryMonitorBlockInputStream")} { Settings insert_settings; @@ -690,7 +689,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map readHeader(in, insert_settings, insert_query, client_info, log); CompressedReadBuffer decompressing_in(in); - NativeBlockInputStream block_in(decompressing_in, ClickHouseRevision::get()); + NativeBlockInputStream block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION); block_in.readPrefix(); while (Block block = block_in.read()) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 172a398258f..f08cdf76cbf 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -583,16 +582,16 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: { WriteBufferFromFile out{first_file_tmp_path}; CompressedWriteBuffer compress{out}; - NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; + NativeBlockOutputStream stream{compress, DBMS_TCP_PROTOCOL_VERSION, block.cloneEmpty()}; /// Prepare the header. /// We wrap the header into a string for compatibility with older versions: /// a shard will able to read the header partly and ignore other parts based on its version. WriteBufferFromOwnString header_buf; - writeVarUInt(ClickHouseRevision::get(), header_buf); + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, header_buf); writeStringBinary(query_string, header_buf); context.getSettingsRef().write(header_buf); - context.getClientInfo().write(header_buf, ClickHouseRevision::get()); + context.getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION); /// Add new fields here, for example: /// writeVarUInt(my_new_data, header_buf); diff --git a/src/Storages/System/StorageSystemProcesses.cpp b/src/Storages/System/StorageSystemProcesses.cpp index c65a6b78e41..d899a1708bf 100644 --- a/src/Storages/System/StorageSystemProcesses.cpp +++ b/src/Storages/System/StorageSystemProcesses.cpp @@ -91,7 +91,7 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex res_columns[i++]->insert(process.client_info.os_user); res_columns[i++]->insert(process.client_info.client_hostname); res_columns[i++]->insert(process.client_info.client_name); - res_columns[i++]->insert(process.client_info.client_revision); + res_columns[i++]->insert(process.client_info.client_tcp_protocol_version); res_columns[i++]->insert(process.client_info.client_version_major); res_columns[i++]->insert(process.client_info.client_version_minor); res_columns[i++]->insert(process.client_info.client_version_patch); From fd33fa7f39a1ff63101980009ba037d136adbf34 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 17:55:41 +0300 Subject: [PATCH 118/390] fixup --- src/Server/TCPHandler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 3e1b53a3142..a2618492a8c 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -927,7 +927,7 @@ void TCPHandler::receiveQuery() /// Interserver secret. std::string received_hash; - if (client_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) { readStringBinary(received_hash, *in, 32); } @@ -1019,7 +1019,7 @@ void TCPHandler::receiveUnexpectedQuery() skip_settings.read(*in, settings_format); std::string skip_hash; - bool interserver_secret = client_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET; + bool interserver_secret = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET; if (interserver_secret) readStringBinary(skip_hash, *in, 32); From acff0feb586d5e2fff6a3691c4e39ec1784af8f3 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 18:09:59 +0300 Subject: [PATCH 119/390] slighly updated CMake files --- CMakeLists.txt | 13 ++++---- cmake_files_header.md | 27 +++++++++++++++-- cmake_flags_and_output.md | 63 ++++++++++++++++++++++++++------------- programs/CMakeLists.txt | 6 ++-- 4 files changed, 78 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e120674347a..e924337626c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ if (USE_STATIC_LIBRARIES) endif () # Implies ${WITH_COVERAGE} -option (ENABLE_FUZZING "Enable fuzzy testing using libfuzzer") +option (ENABLE_FUZZING "Fuzzy testing using libfuzzer") if (ENABLE_FUZZING) message (STATUS "Fuzzing instrumentation enabled") @@ -213,6 +213,7 @@ cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_P if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000) option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON) endif() + if(COMPILER_PIPE) set(COMPILER_FLAGS "${COMPILER_FLAGS} -pipe") else() @@ -249,7 +250,7 @@ if (COMPILER_GCC OR COMPILER_CLANG) endif () # Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc -option(WITH_COVERAGE "Profile the resulting binary/binaries") +option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF) if (WITH_COVERAGE AND COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") @@ -286,7 +287,7 @@ if (COMPILER_CLANG) # https://clang.llvm.org/docs/ThinLTO.html # Applies to clang only. # Disabled when building with tests or sanitizers. - option(ENABLE_THINLTO ON) + option(ENABLE_THINLTO "Clang-specific link time optimization" ON) endif() # We cannot afford to use LTO when compiling unit tests, and it's not enough @@ -312,12 +313,14 @@ if (COMPILER_CLANG) endif () find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") + if (LLVM_RANLIB_PATH) message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.") set (CMAKE_RANLIB ${LLVM_RANLIB_PATH}) else () message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.") endif () + elseif (ENABLE_THINLTO) message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang") endif () @@ -328,7 +331,7 @@ option(ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) # system. # This mode exists for enthusiastic developers who are searching for trouble. # Useful for maintainers of OS packages. -option (UNBUNDLED "Use system libraries instead of ones in contrib/") +option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF) if (UNBUNDLED) set(NOT_UNBUNDLED OFF) @@ -338,7 +341,7 @@ endif () if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN)) # Using system libs can cause a lot of warnings in includes (on macro expansion). - option(WERROR "Enable -Werror compiler option") + option(WERROR "Enable -Werror compiler option" OFF) else () option(WERROR "Enable -Werror compiler option" ON) endif () diff --git a/cmake_files_header.md b/cmake_files_header.md index 333a1bf5f30..448ae44c6b5 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -1,8 +1,29 @@ -# All about CMake in ClickHouse +# CMake in ClickHouse -## How ClickHouse uses CMake +## TL; DR How to make ClickHouse compile and link faster? -TODO describe separate cmake files for contrib + arch-dependent ones + options finding. +Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. + +``` +cmake .. + -DCMAKE_BUILD_TYPE=Debug + -DENABLE_CLICKHOUSE_ALL=OFF + -DENABLE_CLICKHOUSE_SERVER=ON + -DENABLE_CLICKHOUSE_CLIENT=ON + -DUSE_STATIC_LIBRARIES=OFF + -DCLICKHOUSE_SPLIT_BINARY=ON + -DSPLIT_SHARED_LIBRARIES=ON + -DUNBUNDLED=ON + -DENABLE_UTILS=OFF + -DENABLE_TESTS=OFF +``` + +## CMake files types + +1. ClickHouse's source CMake files (located in the root directory and in `/src`). +2. Arch-dependent CMake files (located in `/cmake/*os_name*`). +3. Libraries finders (search for contrib libraries, located in `/cmake/find`). +3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) ## List of CMake flags diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 4731b12efa2..906a1fa0dbb 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -1,8 +1,29 @@ -# All about CMake in ClickHouse +# CMake in ClickHouse -## How ClickHouse uses CMake +## TL; DR How to make ClickHouse compile and link faster? -TODO describe separate cmake files for contrib + arch-dependent ones + options finding. +Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. + +``` +cmake .. + -DCMAKE_BUILD_TYPE=Debug + -DENABLE_CLICKHOUSE_ALL=OFF + -DENABLE_CLICKHOUSE_SERVER=ON + -DENABLE_CLICKHOUSE_CLIENT=ON + -DUSE_STATIC_LIBRARIES=OFF + -DCLICKHOUSE_SPLIT_BINARY=ON + -DSPLIT_SHARED_LIBRARIES=ON + -DUNBUNDLED=ON + -DENABLE_UTILS=OFF + -DENABLE_TESTS=OFF +``` + +## CMake files types + +1. ClickHouse's source CMake files (located in the root directory and in `/src`). +2. Arch-dependent CMake files (located in `/cmake/*os_name*`). +3. Libraries finders (search for contrib libraries, located in `/cmake/find`). +3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) ## List of CMake flags @@ -12,10 +33,10 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L191) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L189) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L226) | `OFF` | Add -march=native compiler flag | | | [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L102) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L215) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L213) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | | [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | | [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | | [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | @@ -25,21 +46,21 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | | [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2) | `OFF` | Use 'clang-tidy' static analyzer if present | | | [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | Enable all ClickHouse tools by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | | [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | | [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | | [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | | [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | | [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | | [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | | [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L33) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | | [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | | [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | | [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | | [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Enables fuzzing instrumentation | Enable fuzzy testing using libfuzzer. Implies ${WITH_COVERAGE}. | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE}. | | [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | | [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | | [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | @@ -49,7 +70,7 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | | [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | | [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L326) | `ON` | Enable all libraries (Global default switch) | | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L327) | `ON` | Enable all libraries (Global default switch) | | | [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | | [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | @@ -63,11 +84,11 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | | [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | | [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | | Adds a Google.Test target binary containing unit tests.. | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62) | `ON` | | Need cmake 3.9+. Usually impractical.. | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L151) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62) | `ON` | Clang-specific link time optimization | Need cmake 3.9+. Usually impractical.. | | [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | | [`FUZZER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0) | `OFF` | Enable fuzzer: libfuzzer | | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L156) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | | [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42) | `OFF` | Linker name or full path | | | [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | | [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | @@ -76,8 +97,8 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0) | `""` | Enable sanitizer: address, memory, thread, undefined | | | [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L100) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | | [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size.. Tradeoff is the inability to debug some source files with e.g. gdb. (empty stack frames and no local variables).". | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L387) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L388) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | | [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | | [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | | [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | @@ -115,9 +136,9 @@ TODO describe separate cmake files for contrib + arch-dependent ones + options f | [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | | [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94) | `ON` | Disable to use shared libraries | | | [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L342) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L343) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | | [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | Enable fuzzy testing using libfuzzer. | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L118) | `OFF` | Profile the resulting binary/binaries | | ## Developer's guide for adding new CMake options @@ -136,13 +157,15 @@ Better: option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) ``` -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, leave a comment above -the `option()` line and explain what it does. The best way would be linking the docs page (if it exists). +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some +pre-conditions, leave a comment above the `option()` line and explain what it does. +The best way would be linking the docs page (if it exists). The comment is parsed into a separate column (see below). Even better: ```cmake +# implies ${TESTS_ARE_ENABLED} # see tests/CMakeLists.txt for implementation detail. option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests") ``` diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 5d07f0bf6e1..ad9e4395e69 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -10,10 +10,10 @@ option (ENABLE_CLICKHOUSE_SERVER ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_CLIENT ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ -option (ENABLE_CLICKHOUSE_LOCAL ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ -option (ENABLE_CLICKHOUSE_BENCHMARK ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_BENCHMARK "Queries benchmarking mode" ${ENABLE_CLICKHOUSE_ALL}) # ??? option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG ${ENABLE_CLICKHOUSE_ALL}) @@ -22,7 +22,7 @@ option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_COMPRESSOR ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ -option (ENABLE_CLICKHOUSE_COPIER ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_COPIER "Inter-cluster data copying mode" ${ENABLE_CLICKHOUSE_ALL}) # ??? option (ENABLE_CLICKHOUSE_FORMAT ${ENABLE_CLICKHOUSE_ALL}) From ac606dca77859cf44ed98020010aafdf35a88cca Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 18:37:23 +0300 Subject: [PATCH 120/390] updated cmake/ files, added some comments --- CMakeLists.txt | 6 +++-- cmake/analysis.cmake | 18 ++++++++++---- cmake/fuzzer.cmake | 8 +++---- cmake/limit_jobs.cmake | 17 ++++++++++--- cmake/sanitize.cmake | 3 ++- cmake/tools.cmake | 2 ++ cmake/warnings.cmake | 5 ++-- cmake_files_header.md | 1 + cmake_flags_and_output.md | 50 +++++++++++++++++++-------------------- 9 files changed, 68 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e924337626c..fbd6b921289 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ endforeach() project(ClickHouse) +# If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) @@ -117,7 +118,7 @@ if (USE_STATIC_LIBRARIES) endif () # Implies ${WITH_COVERAGE} -option (ENABLE_FUZZING "Fuzzy testing using libfuzzer") +option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF) if (ENABLE_FUZZING) message (STATUS "Fuzzing instrumentation enabled") @@ -325,7 +326,8 @@ elseif (ENABLE_THINLTO) message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang") endif () -option(ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) +# Turns on all external libs like s3, kafka, ODBC, ... +option(ENABLE_LIBRARIES "Enable all external libraries by default" ON) # We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your # system. diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake index daaa730ac4b..0818d608f32 100644 --- a/cmake/analysis.cmake +++ b/cmake/analysis.cmake @@ -1,20 +1,28 @@ -# This file configures static analysis tools that can be integrated to the build process +# https://clang.llvm.org/extra/clang-tidy/ +option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF) -option (ENABLE_CLANG_TIDY "Use 'clang-tidy' static analyzer if present" OFF) if (ENABLE_CLANG_TIDY) if (${CMAKE_VERSION} VERSION_LESS "3.6.0") message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") endif() find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") + if (CLANG_TIDY_PATH) - message(STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}. The checks will be run during build process. See the .clang-tidy file at the root directory to configure the checks.") - set (USE_CLANG_TIDY 1) + message(STATUS + "Using clang-tidy: ${CLANG_TIDY_PATH}. + The checks will be run during build process. + See the .clang-tidy file at the root directory to configure the checks.") + + set (USE_CLANG_TIDY ON) + # The variable CMAKE_CXX_CLANG_TIDY will be set inside src and base directories with non third-party code. # set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") elseif (FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) message(FATAL_ERROR "clang-tidy is not found") else () - message(STATUS "clang-tidy is not found. This is normal - the tool is only used for static code analysis and isn't essential for the build.") + message(STATUS + "clang-tidy is not found. + This is normal - the tool is only used for code static analysis and isn't essential for the build.") endif () endif () diff --git a/cmake/fuzzer.cmake b/cmake/fuzzer.cmake index 7ce4559ffae..578a9757270 100644 --- a/cmake/fuzzer.cmake +++ b/cmake/fuzzer.cmake @@ -1,11 +1,12 @@ -option (FUZZER "Enable fuzzer: libfuzzer") - +# see ./CMakeLists.txt for variable declaration if (FUZZER) if (FUZZER STREQUAL "libfuzzer") # NOTE: Eldar Zaitov decided to name it "libfuzzer" instead of "fuzzer" to keep in mind another possible fuzzer backends. - # NOTE: no-link means that all the targets are built with instrumentation for fuzzer, but only some of them (tests) have entry point for fuzzer and it's not checked. + # NOTE: no-link means that all the targets are built with instrumentation for fuzzer, but only some of them + # (tests) have entry point for fuzzer and it's not checked. set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=fuzzer-no-link") endif() @@ -14,7 +15,6 @@ if (FUZZER) if (NOT LIB_FUZZING_ENGINE) set (LIB_FUZZING_ENGINE "-fsanitize=fuzzer") endif () - else () message (FATAL_ERROR "Unknown fuzzer type: ${FUZZER}") endif () diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 4f305bfb4c3..5b962f34c38 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -6,26 +6,35 @@ cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES) -option(PARALLEL_COMPILE_JOBS "Define the maximum number of concurrent compilation jobs" "") +# 1 if not set +option(PARALLEL_COMPILE_JOBS "Maximum number of concurrent compilation jobs" "") + +# 1 if not set +option(PARALLEL_LINK_JOBS "Maximum number of concurrent link jobs" "") + if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY) math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY}) + if (NOT PARALLEL_COMPILE_JOBS) set (PARALLEL_COMPILE_JOBS 1) endif () endif () + if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)) set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR}) string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE}) set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS}) endif () -option(PARALLEL_LINK_JOBS "Define the maximum number of concurrent link jobs" "") + if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY) math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) + if (NOT PARALLEL_LINK_JOBS) set (PARALLEL_LINK_JOBS 1) endif () endif () + if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)) set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR}) string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK}) @@ -33,5 +42,7 @@ if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS endif () if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS) - message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}") + message(STATUS + "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. + Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}") endif () diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 32443ed78c3..45f2acec53a 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -1,4 +1,5 @@ -option (SANITIZE "Enable sanitizer: address, memory, thread, undefined" "") +# Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) +option (SANITIZE "Enable one of the code sanitizers" "") set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index a6ea573a59d..730b396d4ff 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -40,7 +40,9 @@ endif () STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) +# Example values: lld-10, gold option (LINKER_NAME "Linker name or full path") + if (COMPILER_GCC AND NOT LINKER_NAME) find_program (LLD_PATH NAMES "ld.lld") find_program (GOLD_PATH NAMES "ld.gold") diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index 2f78dc34079..1a6fc27d238 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -17,8 +17,9 @@ if (USE_DEBUG_HELPERS) endif () # Add some warnings that are not available even with -Wall -Wextra -Wpedantic. - -option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON) +# Intended for exploration of new compiler warnings that may be found useful. +# Applies to clang only +option (WEVERYTHING "Enable -Weverything option with some exceptions." ON) # Control maximum size of stack frames. It can be important if the code is run in fibers with small stack size. # Only in release build because debug has too large stack frames. diff --git a/cmake_files_header.md b/cmake_files_header.md index 448ae44c6b5..f2ebad8cc30 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -14,6 +14,7 @@ cmake .. -DCLICKHOUSE_SPLIT_BINARY=ON -DSPLIT_SHARED_LIBRARIES=ON -DUNBUNDLED=ON + -DENABLE_LIBRARIES=OFF -DENABLE_UTILS=OFF -DENABLE_TESTS=OFF ``` diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 906a1fa0dbb..7146c893134 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -14,6 +14,7 @@ cmake .. -DCLICKHOUSE_SPLIT_BINARY=ON -DSPLIT_SHARED_LIBRARIES=ON -DUNBUNDLED=ON + -DENABLE_LIBRARIES=OFF -DENABLE_UTILS=OFF -DENABLE_TESTS=OFF ``` @@ -33,10 +34,10 @@ cmake .. | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L189) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L226) | `OFF` | Add -march=native compiler flag | | -| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L102) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L213) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L190) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | +| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L214) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | | [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | | [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | | [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | @@ -44,7 +45,7 @@ cmake .. | [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | | [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | | [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L2) | `OFF` | Use 'clang-tidy' static analyzer if present | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/. | | [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | Enable all ClickHouse tools by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | | [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | | [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | @@ -60,7 +61,7 @@ cmake .. | [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | | [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | | [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE}. | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE}. | | [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | | [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | | [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | @@ -68,9 +69,9 @@ cmake .. | [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | | [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | | [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | | [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L327) | `ON` | Enable all libraries (Global default switch) | | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, .... | | [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | | [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | @@ -84,21 +85,20 @@ cmake .. | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | | [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | | [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L151) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L62) | `ON` | Clang-specific link time optimization | Need cmake 3.9+. Usually impractical.. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L30) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | | -| [`FUZZER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/fuzzer.cmake#L0) | `OFF` | Enable fuzzer: libfuzzer | | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L156) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L42) | `OFF` | Linker name or full path | | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L152) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+. Usually impractical.. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold. | | [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L8) | `""` | Define the maximum number of concurrent compilation jobs | | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L21) | `""` | Define the maximum number of concurrent link jobs | | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L0) | `""` | Enable sanitizer: address, memory, thread, undefined | | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L100) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | +| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set. | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set. | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing). | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | | [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size.. Tradeoff is the inability to debug some source files with e.g. gdb. (empty stack frames and no local variables).". | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L388) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | | [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | | [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | | [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | @@ -134,11 +134,11 @@ cmake .. | [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | | [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | | [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | -| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L94) | `ON` | Disable to use shared libraries | | +| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | | [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L343) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | -| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L20) | `ON` | Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang. | | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L118) | `OFF` | Profile the resulting binary/binaries | | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | +| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic.. Intended for exploration of new compiler warnings that may be found useful.. Applies to clang only. | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | ## Developer's guide for adding new CMake options From 7eff17cd408d6c432ee42e1aee70d75562196fca Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 18:46:55 +0300 Subject: [PATCH 121/390] removed extra dots in generated list --- cmake_files_header.md | 2 +- cmake_flags_and_output.md | 116 +++++++++++++++++++------------------- cmake_flags_generator.py | 4 +- programs/CMakeLists.txt | 3 +- 4 files changed, 63 insertions(+), 62 deletions(-) diff --git a/cmake_files_header.md b/cmake_files_header.md index f2ebad8cc30..4cfd41e2964 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -4,7 +4,7 @@ Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. -``` +```cmake cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_CLICKHOUSE_ALL=OFF diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 7146c893134..3b08bcc60de 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -4,7 +4,7 @@ Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. -``` +```cmake cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_CLICKHOUSE_ALL=OFF @@ -38,67 +38,67 @@ cmake .. | [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | | [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | | [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L214) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | | [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/. | -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L6) | `ON` | Enable all ClickHouse tools by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.),. each of them may be built and linked as a separate library.. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L15) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/. | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L21) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L24) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/. | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L18) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L36) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L12) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/. | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L30) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L33) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ???. | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L8) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L28) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L37) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L34) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | | [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | | [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE}. | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE} | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | | [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | | [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+. Usually impractical.. See also ENABLE_THINLTO. | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+ Usually impractical. See also ENABLE_THINLTO | | [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, .... | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | | [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | | [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | | [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | | [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | | [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L152) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+. Usually impractical.. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.. | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64.. Implies ${ENABLE_FASTMEMCPY}. | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold. | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+ Usually impractical. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies ${ENABLE_FASTMEMCPY} | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold | | [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | | [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set. | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set. | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing). | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY.. Faster linking if turned on.. | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size.. Tradeoff is the inability to debug some source files with e.g. gdb. (empty stack frames and no local variables).". | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | | [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use. | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | | [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | | [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | | [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | @@ -113,31 +113,31 @@ cmake .. | [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | | [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | | [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | +| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | | [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | | [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | | [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | | [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | | [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | | [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L0) | `ON` | Use internal Poco library | | +| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | | [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | | [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | | [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | +| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | | [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L0) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | +| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | | [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L0) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | | [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | | [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L0) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion).. | -| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic.. Intended for exploration of new compiler warnings that may be found useful.. Applies to clang only. | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | +| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | | [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | ## Developer's guide for adding new CMake options diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 79da97f218b..4fbde03b347 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -44,7 +44,7 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **opt anchor=make_anchor(_name), name=_name, path=path, - line=line) + line=line if line > 0 else 1) if options.get("no_desc", False): description: str = "" @@ -69,7 +69,7 @@ def process_file(input_name: str, **options) -> None: if not re.match("\s*#\s*", maybe_comment_line): break - comment = re.sub("\s*#\s*", "", maybe_comment_line) + ". " + comment + comment = re.sub("\s*#\s*", "", maybe_comment_line) + " " + comment return n, comment diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index ad9e4395e69..a440d54d688 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -4,7 +4,8 @@ endif () # The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. -option (ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse tools by default" ON) +# If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. +option (ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse modes by default" ON) option (ENABLE_CLICKHOUSE_SERVER ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_CLIENT ${ENABLE_CLICKHOUSE_ALL}) From 507acdb5e7977fcaa0390c324fd29f504461ab66 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 19:17:56 +0300 Subject: [PATCH 122/390] fix --- programs/CMakeLists.txt | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 6057db5a6f9..b6e2bb035bb 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,7 +2,6 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -<<<<<<< HEAD # The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. # If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. @@ -34,24 +33,6 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR ${ENABLE_CLICKHOUSE_ALL}) # ??? option (ENABLE_CLICKHOUSE_ODBC_BRIDGE ${ENABLE_CLICKHOUSE_ALL}) -======= -# 'clickhouse' binary is a multi purpose tool, -# that contain multiple execution modes (client, server, etc.) -# each of them is built and linked as a separate library, defined below. - -option (ENABLE_CLICKHOUSE_ALL "Enable all tools" ON) -option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_GIT_IMPORT "Enable clickhouse-git-import" ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL}) ->>>>>>> upstream/master if (CLICKHOUSE_SPLIT_BINARY) option(ENABLE_CLICKHOUSE_INSTALL) From 04ac8e3b98f116aa1600398d558b78460612652a Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 19:19:14 +0300 Subject: [PATCH 123/390] removed -DUNBUNDLED line --- cmake_files_header.md | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake_files_header.md b/cmake_files_header.md index 4cfd41e2964..63e1b539602 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -13,7 +13,6 @@ cmake .. -DUSE_STATIC_LIBRARIES=OFF -DCLICKHOUSE_SPLIT_BINARY=ON -DSPLIT_SHARED_LIBRARIES=ON - -DUNBUNDLED=ON -DENABLE_LIBRARIES=OFF -DENABLE_UTILS=OFF -DENABLE_TESTS=OFF From a8fa23fb9dbb4db8f6f960f89469798088d8b280 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 17 Sep 2020 19:27:51 +0300 Subject: [PATCH 124/390] updated multi-line command --- cmake_files_header.md | 22 ++++++++++++---------- cmake_flags_and_output.md | 23 ++++++++++++----------- programs/CMakeLists.txt | 2 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/cmake_files_header.md b/cmake_files_header.md index 63e1b539602..17a409c0b29 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -5,16 +5,18 @@ Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. ```cmake -cmake .. - -DCMAKE_BUILD_TYPE=Debug - -DENABLE_CLICKHOUSE_ALL=OFF - -DENABLE_CLICKHOUSE_SERVER=ON - -DENABLE_CLICKHOUSE_CLIENT=ON - -DUSE_STATIC_LIBRARIES=OFF - -DCLICKHOUSE_SPLIT_BINARY=ON - -DSPLIT_SHARED_LIBRARIES=ON - -DENABLE_LIBRARIES=OFF - -DENABLE_UTILS=OFF +cmake .. \ + -DCMAKE_C_COMPILER=/bin/clang-10 \ + -DCMAKE_CXX_COMPILER=/bin/clang++-10 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DENABLE_CLICKHOUSE_ALL=OFF \ + -DENABLE_CLICKHOUSE_SERVER=ON \ + -DENABLE_CLICKHOUSE_CLIENT=ON \ + -DUSE_STATIC_LIBRARIES=OFF \ + -DCLICKHOUSE_SPLIT_BINARY=ON \ + -DSPLIT_SHARED_LIBRARIES=ON \ + -DENABLE_LIBRARIES=OFF \ + -DENABLE_UTILS=OFF \ -DENABLE_TESTS=OFF ``` diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 3b08bcc60de..4f3dc332648 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -5,17 +5,18 @@ Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. ```cmake -cmake .. - -DCMAKE_BUILD_TYPE=Debug - -DENABLE_CLICKHOUSE_ALL=OFF - -DENABLE_CLICKHOUSE_SERVER=ON - -DENABLE_CLICKHOUSE_CLIENT=ON - -DUSE_STATIC_LIBRARIES=OFF - -DCLICKHOUSE_SPLIT_BINARY=ON - -DSPLIT_SHARED_LIBRARIES=ON - -DUNBUNDLED=ON - -DENABLE_LIBRARIES=OFF - -DENABLE_UTILS=OFF +cmake .. \ + -DCMAKE_C_COMPILER=/bin/clang-10 \ + -DCMAKE_CXX_COMPILER=/bin/clang++-10 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DENABLE_CLICKHOUSE_ALL=OFF \ + -DENABLE_CLICKHOUSE_SERVER=ON \ + -DENABLE_CLICKHOUSE_CLIENT=ON \ + -DUSE_STATIC_LIBRARIES=OFF \ + -DCLICKHOUSE_SPLIT_BINARY=ON \ + -DSPLIT_SHARED_LIBRARIES=ON \ + -DENABLE_LIBRARIES=OFF \ + -DENABLE_UTILS=OFF \ -DENABLE_TESTS=OFF ``` diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index b6e2bb035bb..9ed54eb9acd 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -35,7 +35,7 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_ODBC_BRIDGE ${ENABLE_CLICKHOUSE_ALL}) if (CLICKHOUSE_SPLIT_BINARY) - option(ENABLE_CLICKHOUSE_INSTALL) + option(ENABLE_CLICKHOUSE_INSTALL "???" OFF) else () option(ENABLE_CLICKHOUSE_INSTALL ${ENABLE_CLICKHOUSE_ALL}) endif () From c9baceb7605799dca12ed9450fc2ec230baed523 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 17 Sep 2020 21:07:14 +0300 Subject: [PATCH 125/390] Change thread name for DDLWorker executor threads --- src/Interpreters/DDLWorker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 526f15d921f..5b346eec54a 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -515,6 +515,7 @@ void DDLWorker::scheduleTasks() { worker_pool.scheduleOrThrowOnError([this, task_ptr = task.release()]() { + setThreadName("DDLWorkerExec"); enqueueTask(DDLTaskPtr(task_ptr)); }); } From a783fe110f0e201bb71a934b7680cd09d219e562 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 21:15:19 +0300 Subject: [PATCH 126/390] Fix build --- src/Common/getMappedArea.cpp | 4 ++-- src/Common/getMappedArea.h | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp index 6817bea8cb1..018db16038b 100644 --- a/src/Common/getMappedArea.cpp +++ b/src/Common/getMappedArea.cpp @@ -1,3 +1,5 @@ +#include "getMappedArea.h" + #if defined(__linux__) #include @@ -6,8 +8,6 @@ #include #include -#include "getMappedArea.h" - namespace DB { diff --git a/src/Common/getMappedArea.h b/src/Common/getMappedArea.h index 7d120f0d9a7..3317f72bdc9 100644 --- a/src/Common/getMappedArea.h +++ b/src/Common/getMappedArea.h @@ -1,4 +1,6 @@ #include +#include + namespace DB { From 7fb4dfea2c625497a13417a8500c3dc58dc1784b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 22:50:43 +0300 Subject: [PATCH 127/390] Small improvements for IStorage::rename --- src/Storages/StorageDistributed.cpp | 4 ++-- src/Storages/StorageFile.cpp | 5 ++++- src/Storages/StorageLog.cpp | 17 ++++++++++------- src/Storages/StorageStripeLog.cpp | 11 +++++++---- src/Storages/StorageTinyLog.cpp | 15 +++++++++------ 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 596681002ab..073dd7cc754 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -866,7 +866,7 @@ void StorageDistributed::flushClusterNodesAllData() void StorageDistributed::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - if (!relative_data_path.empty()) + if (!relative_data_path.empty() && relative_data_path != new_path_to_table_data) renameOnDisk(new_path_to_table_data); renameInMemory(new_table_id); } @@ -878,7 +878,7 @@ void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) { const String path(disk->getPath()); auto new_path = path + new_path_to_table_data; - Poco::File(path + relative_data_path).renameTo(new_path); + disk->moveDirectory(path + relative_data_path, new_path); LOG_DEBUG(log, "Updating path to {}", new_path); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 9e60d5bad15..cc47047dc78 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -525,9 +525,12 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (paths.size() != 1) throw Exception("Can't rename table " + getStorageID().getNameForLogs() + " in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); + std::string path_new = getTablePath(base_path + new_path_to_table_data, format_name); + if (path_new == paths[0]) + return; + std::unique_lock lock(rwlock); - std::string path_new = getTablePath(base_path + new_path_to_table_data, format_name); Poco::File(Poco::Path(path_new).parent()).createDirectories(); Poco::File(paths[0]).renameTo(path_new); diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 45ab3293723..8c76085e50c 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -548,17 +548,20 @@ void StorageLog::loadMarks() void StorageLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - std::unique_lock lock(rwlock); + if (table_path != new_path_to_table_data) + { + std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); + disk->moveDirectory(table_path, new_path_to_table_data); - table_path = new_path_to_table_data; - file_checker.setPath(table_path + "sizes.json"); + table_path = new_path_to_table_data; + file_checker.setPath(table_path + "sizes.json"); - for (auto & file : files) - file.second.data_file_path = table_path + fileName(file.second.data_file_path); + for (auto & file : files) + file.second.data_file_path = table_path + fileName(file.second.data_file_path); - marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME; + marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME; + } renameInMemory(new_table_id); } diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 3e7fb5b853e..c2bbb4e4bd8 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -282,12 +282,15 @@ StorageStripeLog::StorageStripeLog( void StorageStripeLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - std::unique_lock lock(rwlock); + if (table_path != new_path_to_table_data) + { + std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); + disk->moveDirectory(table_path, new_path_to_table_data); - table_path = new_path_to_table_data; - file_checker.setPath(table_path + "sizes.json"); + table_path = new_path_to_table_data; + file_checker.setPath(table_path + "sizes.json"); + } renameInMemory(new_table_id); } diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index e7fe7e2d5f9..19b6d52b012 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -407,15 +407,18 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type void StorageTinyLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - std::unique_lock lock(rwlock); + if (table_path != new_path_to_table_data) + { + std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); + disk->moveDirectory(table_path, new_path_to_table_data); - table_path = new_path_to_table_data; - file_checker.setPath(table_path + "sizes.json"); + table_path = new_path_to_table_data; + file_checker.setPath(table_path + "sizes.json"); - for (auto & file : files) - file.second.data_file_path = table_path + fileName(file.second.data_file_path); + for (auto & file : files) + file.second.data_file_path = table_path + fileName(file.second.data_file_path); + } renameInMemory(new_table_id); } From 1752bbc1edbebd69369e822dfbf3a544235387e5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 01:29:15 +0300 Subject: [PATCH 128/390] Fix "Arcadia" build --- src/Columns/ya.make | 2 ++ src/Common/ya.make | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Columns/ya.make b/src/Columns/ya.make index 910c479c2a9..78c0e1b992d 100644 --- a/src/Columns/ya.make +++ b/src/Columns/ya.make @@ -2,6 +2,8 @@ LIBRARY() ADDINCL( + contrib/libs/icu/common + contrib/libs/icu/i18n contrib/libs/pdqsort ) diff --git a/src/Common/ya.make b/src/Common/ya.make index 72f1fa42756..9e4f4cd76c0 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -50,6 +50,7 @@ SRCS( formatIPv6.cpp formatReadable.cpp getExecutablePath.cpp + getMappedArea.cpp getMultipleKeysFromConfig.cpp getNumberOfPhysicalCPUCores.cpp hasLinuxCapability.cpp @@ -87,6 +88,7 @@ SRCS( StatusFile.cpp StatusInfo.cpp Stopwatch.cpp + StringRef.cpp StringUtils/StringUtils.cpp StudentTTest.cpp SymbolIndex.cpp From 0ad44b821e6f8417075d26700bbea576cefd272c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 01:45:12 +0300 Subject: [PATCH 129/390] Fix build --- src/Parsers/formatSettingName.cpp | 1 + src/Parsers/formatSettingName.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Parsers/formatSettingName.cpp b/src/Parsers/formatSettingName.cpp index 3f30142716e..c305496fdb3 100644 --- a/src/Parsers/formatSettingName.cpp +++ b/src/Parsers/formatSettingName.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB diff --git a/src/Parsers/formatSettingName.h b/src/Parsers/formatSettingName.h index c9ed94dcc7d..a700d347a5f 100644 --- a/src/Parsers/formatSettingName.h +++ b/src/Parsers/formatSettingName.h @@ -1,5 +1,6 @@ #pragma once +#include #include From d1e90e0e4bbb3d273ae06554991a36613e3844cd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 18 Sep 2020 03:05:51 +0300 Subject: [PATCH 130/390] fix tests --- .../test_adaptive_granularity/test.py | 14 ----------- tests/integration/test_ttl_replicated/test.py | 25 +++---------------- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/tests/integration/test_adaptive_granularity/test.py b/tests/integration/test_adaptive_granularity/test.py index 524c8215081..ec3169bb995 100644 --- a/tests/integration/test_adaptive_granularity/test.py +++ b/tests/integration/test_adaptive_granularity/test.py @@ -274,7 +274,6 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node): "INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)") def callback(n): -<<<<<<< HEAD new_config = """ 1 @@ -283,12 +282,6 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node): n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", new_config) n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", new_config) -======= - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", - "1") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", - "1") ->>>>>>> upstream/master node.restart_with_latest_version(callback_onstop=callback) node.query("SYSTEM RELOAD CONFIG") @@ -331,7 +324,6 @@ def test_version_update_two_nodes(start_dynamic_cluster): assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n' def callback(n): -<<<<<<< HEAD new_config = """ 0 @@ -340,12 +332,6 @@ def test_version_update_two_nodes(start_dynamic_cluster): n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", new_config) n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", new_config) -======= - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", - "0") - n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", - "0") ->>>>>>> upstream/master node12.restart_with_latest_version(callback_onstop=callback) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index b8832126b0b..878db2da11f 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -55,20 +55,12 @@ def test_merge_with_ttl_timeout(started_cluster): drop_table([node1, node2], table) for node in [node1, node2]: node.query( -<<<<<<< HEAD - ''' - CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date) - SETTINGS min_bytes_for_wide_part=0; - '''.format(replica=node.name, table=table)) -======= ''' CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') - ORDER BY id PARTITION BY toDayOfMonth(date); + ORDER BY id PARTITION BY toDayOfMonth(date) + SETTINGS min_bytes_for_wide_part=0; '''.format(replica=node.name, table=table)) ->>>>>>> upstream/master node1.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) node2.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) @@ -256,17 +248,6 @@ limitations under the License.""" time.sleep(0.5) node1.query( -<<<<<<< HEAD - """ - CREATE TABLE {name} ( - s1 String, - d1 DateTime - ) ENGINE = {engine} - ORDER BY tuple() - TTL d1 + INTERVAL 1 DAY DELETE - SETTINGS min_bytes_for_wide_part=0 - """.format(name=name, engine=engine)) -======= """ CREATE TABLE {name} ( s1 String, @@ -274,8 +255,8 @@ limitations under the License.""" ) ENGINE = {engine} ORDER BY tuple() TTL d1 + INTERVAL 1 DAY DELETE + SETTINGS min_bytes_for_wide_part=0 """.format(name=name, engine=engine)) ->>>>>>> upstream/master node1.query("""ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format(name=name)) node1.query("""ALTER TABLE {name} ADD COLUMN b1 Int32""".format(name=name)) From e6914acb3b9b93d26c38d7bac5634fd97f14c37b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 18 Sep 2020 10:44:17 +0300 Subject: [PATCH 131/390] Fix check for merge tree settings on server startup --- src/Storages/MergeTree/MergeTreeSettings.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 7f537ec330a..69a7b6d4eb9 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -77,24 +77,24 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) void MergeTreeSettings::sanityCheck(const Settings & query_settings) const { - if (number_of_free_entries_in_pool_to_execute_mutation >= query_settings.background_pool_size) + if (number_of_free_entries_in_pool_to_execute_mutation > query_settings.background_pool_size) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_execute_mutation' setting" " ({}) (default values are defined in section of config.xml" " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" - " is greater or equals to the value of 'background_pool_size'" + " is greater the value of 'background_pool_size'" " ({}) (the value is defined in users.xml for default profile)." " This indicates incorrect configuration because mutations cannot work with these settings.", number_of_free_entries_in_pool_to_execute_mutation, query_settings.background_pool_size); } - if (number_of_free_entries_in_pool_to_lower_max_size_of_merge >= query_settings.background_pool_size) + if (number_of_free_entries_in_pool_to_lower_max_size_of_merge > query_settings.background_pool_size) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting" " ({}) (default values are defined in section of config.xml" " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" - " is greater or equals to the value of 'background_pool_size'" + " is greater the value of 'background_pool_size'" " ({}) (the value is defined in users.xml for default profile)." " This indicates incorrect configuration because the maximum size of merge will be always lowered.", number_of_free_entries_in_pool_to_lower_max_size_of_merge, From 96a693baec89f8feb7589c96dfee950481887f1a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 18 Sep 2020 10:46:23 +0300 Subject: [PATCH 132/390] Better message --- src/Storages/MergeTree/MergeTreeSettings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 69a7b6d4eb9..00580c8d8bb 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -82,7 +82,7 @@ void MergeTreeSettings::sanityCheck(const Settings & query_settings) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_execute_mutation' setting" " ({}) (default values are defined in section of config.xml" " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" - " is greater the value of 'background_pool_size'" + " is greater than the value of 'background_pool_size'" " ({}) (the value is defined in users.xml for default profile)." " This indicates incorrect configuration because mutations cannot work with these settings.", number_of_free_entries_in_pool_to_execute_mutation, @@ -94,7 +94,7 @@ void MergeTreeSettings::sanityCheck(const Settings & query_settings) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting" " ({}) (default values are defined in section of config.xml" " or the value can be specified per table in SETTINGS section of CREATE TABLE query)" - " is greater the value of 'background_pool_size'" + " is greater than the value of 'background_pool_size'" " ({}) (the value is defined in users.xml for default profile)." " This indicates incorrect configuration because the maximum size of merge will be always lowered.", number_of_free_entries_in_pool_to_lower_max_size_of_merge, From 1d47a87301bbff6007f08e0fa6e9f0ae7453510e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 18 Sep 2020 13:06:25 +0300 Subject: [PATCH 133/390] restart the tests From 449189dcdacaad381fb901a172d90912afdcb6a3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 18 Sep 2020 13:47:09 +0300 Subject: [PATCH 134/390] Initialize global thread pool before we fetch configs from ZK --- programs/server/Server.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index aa947b22593..c158a947ca4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -280,6 +280,11 @@ int Server::main(const std::vector & /*args*/) global_context->makeGlobalContext(); global_context->setApplicationType(Context::ApplicationType::SERVER); + // Initialize global thread pool. Do it before we fetch configs from zookeeper + // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will + // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. + GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 10000)); + bool has_zookeeper = config().has("zookeeper"); zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); }); @@ -414,9 +419,6 @@ int Server::main(const std::vector & /*args*/) DateLUT::instance(); LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); - /// Initialize global thread pool - GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 10000)); - /// Storage with temporary data for processing of heavy queries. { std::string tmp_path = config().getString("tmp_path", path + "tmp/"); From 1ee2e3d2b398546395b0aac5431c98443f48e910 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Fri, 18 Sep 2020 16:03:48 +0300 Subject: [PATCH 135/390] Review fix --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f2010b4e34e..2d1a1ae3a09 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1498,7 +1498,9 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( } else { - // For the case of one continuous range of keys we use binary search algorithm + /// In case when SELECT's predicate defines a single continuous interval of keys, + /// we can use binary search algorithm to find the left and right endpoint key marks of such interval. + /// The returned value is the minumum range of marks, containing all keys for which KeyCondition holds LOG_TRACE(log, "Running binary search on index range for part {} ({} marks)", part->name, marks_count); @@ -1536,8 +1538,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( result_range.end = searched_right; LOG_TRACE(log, "Found (RIGHT) boundary mark: {}", searched_right); - - if (may_be_true_in_range(result_range)) + if (result_range.begin < result_range.end && may_be_true_in_range(result_range)) res.emplace_back(std::move(result_range)); LOG_TRACE(log, "Found {} range in {} steps", res.empty() ? "empty" : "continuous", steps); From 3953ed485ba8cdf7f4f99da055738d250df9c358 Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 18 Sep 2020 16:09:22 +0300 Subject: [PATCH 136/390] added sub-tables in report, replaced some USE_ -> ENABLE_ for consistency --- base/daemon/CMakeLists.txt | 2 +- base/daemon/SentryWriter.cpp | 2 +- cmake/find/cxx.cmake | 11 +- cmake/find/llvm.cmake | 2 +- cmake/find/sentry.cmake | 9 +- cmake/find/simdjson.cmake | 6 +- cmake/find/snappy.cmake | 6 +- cmake/find/unwind.cmake | 4 +- cmake/target.cmake | 2 +- cmake_files_footer.md | 17 +-- cmake_files_header.md | 7 +- cmake_flags_and_output.md | 126 +++++++++--------- cmake_flags_generator.py | 40 +++++- .../utilities/clickhouse-compressor.md | 0 .../utilities/clickhouse-obfuscator.md | 42 ++++++ .../en/operations/utilities/odbc-bridge.md | 0 programs/CMakeLists.txt | 31 +++-- programs/server/Server.cpp | 4 +- src/Common/QueryProfiler.cpp | 4 +- src/Common/QueryProfiler.h | 2 +- src/Common/StackTrace.cpp | 4 +- src/Functions/CMakeLists.txt | 2 +- src/Functions/FunctionsJSON.h | 2 +- src/Functions/SimdJSONParser.h | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- 25 files changed, 201 insertions(+), 128 deletions(-) rename programs/compressor/README.md => docs/en/operations/utilities/clickhouse-compressor.md (100%) create mode 100644 docs/en/operations/utilities/clickhouse-obfuscator.md rename programs/odbc-bridge/README.md => docs/en/operations/utilities/odbc-bridge.md (100%) diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 26d59a57e7f..97278c593c8 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -7,6 +7,6 @@ add_library (daemon target_include_directories (daemon PUBLIC ..) target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) -if (USE_SENTRY) +if (ENABLE_SENTRY) target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) endif () diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 33f2b237dd5..7794be36c73 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -16,7 +16,7 @@ # include #endif -#if USE_SENTRY +#if ENABLE_SENTRY # include // Y_IGNORE # include diff --git a/cmake/find/cxx.cmake b/cmake/find/cxx.cmake index 02f7113e6fb..04bc3caf46a 100644 --- a/cmake/find/cxx.cmake +++ b/cmake/find/cxx.cmake @@ -1,16 +1,19 @@ -option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) +option (ENABLE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) -if (NOT USE_LIBCXX) +if (NOT ENABLE_LIBCXX) if (USE_INTERNAL_LIBCXX_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with USE_LIBCXX=OFF") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with ENABLE_LIBCXX=OFF") endif() + target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) return() endif() set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED}) -option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}) + +option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi libraries instead of bundled" + ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt") if (USE_INTERNAL_LIBCXX_LIBRARY) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 70d0e356c39..e99d0f05c54 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -44,7 +44,7 @@ if (NOT USE_INTERNAL_LLVM_LIBRARY) set (USE_EMBEDDED_COMPILER 0) endif() - if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX) + if (LLVM_FOUND AND OS_LINUX AND ENABLE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX) message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages " "in Linux is incompatible with libc++ ABI. LLVM Will be disabled. Force: -DFORCE_LLVM_WITH_LIBCXX=ON") message (${RECONFIGURE_MESSAGE_LEVEL} "Unsupported LLVM configuration, cannot enable LLVM") diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 2936c045f99..9739023a148 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -1,24 +1,25 @@ set (SENTRY_LIBRARY "sentry") + set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include") if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive") - if (USE_SENTRY) + if (ENABLE_SENTRY) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library") endif() return() endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) - option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) + option (ENABLE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) set (SENTRY_TRANSPORT "curl" CACHE STRING "") set (SENTRY_BACKEND "none" CACHE STRING "") set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") set (SENTRY_PIC OFF CACHE BOOL "") set (BUILD_SHARED_LIBS OFF) - message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") + message (STATUS "Using sentry=${ENABLE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") -elseif (USE_SENTRY) +elseif (ENABLE_SENTRY) message (${RECONFIGURE_MESSAGE_LEVEL} "Sentry is not supported in current configuration") endif () diff --git a/cmake/find/simdjson.cmake b/cmake/find/simdjson.cmake index cffe20bdb2d..f1946e19a28 100644 --- a/cmake/find/simdjson.cmake +++ b/cmake/find/simdjson.cmake @@ -1,11 +1,11 @@ -option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) +option (ENABLE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h") message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init --recursive") - if (USE_SIMDJSON) + if (ENABLE_SIMDJSON) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library") endif() return() endif () -message(STATUS "Using simdjson=${USE_SIMDJSON}") +message(STATUS "Using simdjson=${ENABLE_SIMDJSON}") diff --git a/cmake/find/snappy.cmake b/cmake/find/snappy.cmake index e719231c338..830d54975ef 100644 --- a/cmake/find/snappy.cmake +++ b/cmake/find/snappy.cmake @@ -1,8 +1,8 @@ -option(USE_SNAPPY "Enable support of snappy library" ${ENABLE_LIBRARIES}) +option(ENABLE_SNAPPY "Enable snappy library" ${ENABLE_LIBRARIES}) -if(NOT USE_SNAPPY) +if(NOT ENABLE_SNAPPY) if (USE_INTERNAL_SNAPPY_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with USE_SNAPPY=OFF") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with ENABLE_SNAPPY=OFF") endif() return() endif() diff --git a/cmake/find/unwind.cmake b/cmake/find/unwind.cmake index c9f5f30a5d6..3f8a0af078c 100644 --- a/cmake/find/unwind.cmake +++ b/cmake/find/unwind.cmake @@ -1,6 +1,6 @@ -option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) +option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) -if (USE_UNWIND) +if (ENABLE_UNWIND) add_subdirectory(contrib/libunwind-cmake) set (UNWIND_LIBRARIES unwind) set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) diff --git a/cmake/target.cmake b/cmake/target.cmake index 7174ca3c2a9..832b9db2e4a 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -20,7 +20,7 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") # no protobuf -> no grpc - set (USE_SNAPPY OFF CACHE INTERNAL "") + set (ENABLE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") # no snappy and protobuf -> no parquet set (ENABLE_ORC OFF CACHE INTERNAL "") # no arrow (parquet) -> no orc diff --git a/cmake_files_footer.md b/cmake_files_footer.md index 21cf7677753..ab884bd4dfe 100644 --- a/cmake_files_footer.md +++ b/cmake_files_footer.md @@ -26,13 +26,7 @@ Even better: ```cmake # implies ${TESTS_ARE_ENABLED} # see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests") -``` - -Note that the default value (`OFF`) can be omitted if you provide a description, e.g. - -``` -option(MYOPTION "My description") +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) ``` ### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. @@ -85,7 +79,7 @@ Better: ```cmake # Only applicable for clang. # Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). ``` ### Don't assume other developers know as much as you do. @@ -105,7 +99,7 @@ Better (combined with the above hint): # https://clang.llvm.org/docs/ThinLTO.html # Only applicable for clang. # Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). ``` Other example, bad: @@ -118,13 +112,10 @@ Better: ```cmake # https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE) +option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) ``` ### Prefer consistent default values. CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. Prefer the `ON/OFF` values, if possible. - - - diff --git a/cmake_files_header.md b/cmake_files_header.md index 17a409c0b29..669399e9996 100644 --- a/cmake_files_header.md +++ b/cmake_files_header.md @@ -2,7 +2,7 @@ ## TL; DR How to make ClickHouse compile and link faster? -Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. +Developer only! This command will likely fulfill most of your needs. Run before calling `ninja`. ```cmake cmake .. \ @@ -31,7 +31,4 @@ cmake .. \ * This list is auto-generated by [this bash script](bash.sh). * The flag name is a link to its position in the code. -* If an option's default valus is itself an option, it's also a link to its position in this list. - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| +* If an option's default value is itself an option, it's also a link to its position in this list. diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 4f3dc332648..31614d77a72 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -2,7 +2,7 @@ ## TL; DR How to make ClickHouse compile and link faster? -Developer only! This command will likely fulfil most of your needs. Run before calling `ninja`. +Developer only! This command will likely fulfill most of your needs. Run before calling `ninja`. ```cmake cmake .. \ @@ -31,14 +31,27 @@ cmake .. \ * This list is auto-generated by [this bash script](bash.sh). * The flag name is a link to its position in the code. -* If an option's default valus is itself an option, it's also a link to its position in this list. +* If an option's default value is itself an option, it's also a link to its position in this list. +### ClickHouse modes + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Ddata compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | +### External libraries | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L190) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | -| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L214) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | | [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | | [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | | [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | @@ -47,35 +60,20 @@ cmake .. \ | [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | | [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | | [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L16) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L28) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L37) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L13) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L34) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | ??? | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | | | | [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | | [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | | [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE} | | [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | | [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | | [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | | [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | | [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | | [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | `0` | Enable ICU | | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+ Usually impractical. See also ENABLE_THINLTO | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | | [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | +| [`ENABLE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | | [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | -| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | `FALSE` | Enable MySQL | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | | [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | | [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | | [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | @@ -84,25 +82,20 @@ cmake .. \ | [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | | [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`ENABLE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`ENABLE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | | [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | | [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L152) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+ Usually impractical. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies ${ENABLE_FASTMEMCPY} | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold | -| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | +| [`ENABLE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | +### External libraries system/bundled mode + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | | [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | | [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | `ON` | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | | [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | | [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | | [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | @@ -111,9 +104,8 @@ cmake .. \ | [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | | [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | | [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | -| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | `ON` | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | | [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | | [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | | [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | @@ -131,13 +123,34 @@ cmake .. \ | [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | | [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | | [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L4) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable support of snappy library | | +### Other flags + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L190) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | +| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L214) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE} | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+ Usually impractical. See also ENABLE_THINLTO | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | +| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L152) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+ Usually impractical. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies ${ENABLE_FASTMEMCPY} | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold | +| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | +| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | | [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `ON` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | | [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | | [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | @@ -168,13 +181,7 @@ Even better: ```cmake # implies ${TESTS_ARE_ENABLED} # see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests") -``` - -Note that the default value (`OFF`) can be omitted if you provide a description, e.g. - -``` -option(MYOPTION "My description") +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) ``` ### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. @@ -227,7 +234,7 @@ Better: ```cmake # Only applicable for clang. # Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). ``` ### Don't assume other developers know as much as you do. @@ -247,7 +254,7 @@ Better (combined with the above hint): # https://clang.llvm.org/docs/ThinLTO.html # Only applicable for clang. # Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO ON). +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). ``` Other example, bad: @@ -260,13 +267,10 @@ Better: ```cmake # https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE) +option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) ``` ### Prefer consistent default values. CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. Prefer the `ON/OFF` values, if possible. - - - diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 4fbde03b347..2c79595b564 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -16,8 +16,14 @@ ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" default_anchor_str: str = "[`{name}`](#{anchor})" +table_header: str = """ +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +""" + # Needed to detect conditional variables (those which are defined twice) -entities: Dict[str, str] = {} +# name -> (path, values) +entities: Dict[str, Tuple[str, str]] = {} def make_anchor(t: str) -> str: @@ -51,7 +57,7 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **opt else: description: str = "".join(_description.split("\n")) + " | " - entities[name] = "| " + name + " | " + default + " | " + description + comment + " |" + entities[_name] = path, "| " + name + " | " + default + " | " + description + comment + " |" def process_file(input_name: str, **options) -> None: with open(input_name, 'r') as cmake_file: @@ -97,8 +103,34 @@ def process() -> None: with open(header_file_name, "r") as header: f.write(header.read()) - for k in sorted(entities.keys()): - f.write(entities[k] + "\n") + sorted_keys: List[str] = sorted(entities.keys()) + ignored_keys: List[str] = [] + + f.write("### ClickHouse modes\n" + table_header) + + for k in sorted_keys: + if k.startswith("ENABLE_CLICKHOUSE_"): + f.write(entities[k][1] + "\n") + ignored_keys.append(k) + + f.write("### External libraries\n" + table_header) + + for k in sorted_keys: + if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): + f.write(entities[k][1] + "\n") + ignored_keys.append(k) + + f.write("### External libraries system/bundled mode\n" + table_header) + + for k in sorted_keys: + if k.startswith("USE_INTERNAL_"): + f.write(entities[k][1] + "\n") + ignored_keys.append(k) + + f.write("### Other flags\n" + table_header) + + for k in sorted(set(sorted_keys).difference(set(ignored_keys))): + f.write(entities[k][1] + "\n") with open(footer_file_name, "r") as footer: f.write(footer.read()) diff --git a/programs/compressor/README.md b/docs/en/operations/utilities/clickhouse-compressor.md similarity index 100% rename from programs/compressor/README.md rename to docs/en/operations/utilities/clickhouse-compressor.md diff --git a/docs/en/operations/utilities/clickhouse-obfuscator.md b/docs/en/operations/utilities/clickhouse-obfuscator.md new file mode 100644 index 00000000000..8a2ea1eecf6 --- /dev/null +++ b/docs/en/operations/utilities/clickhouse-obfuscator.md @@ -0,0 +1,42 @@ +# ClickHouse obfuscator + +Simple tool for table data obfuscation. + +It reads input table and produces output table, that retain some properties of input, but contains different data. +It allows to publish almost real production data for usage in benchmarks. + +It is designed to retain the following properties of data: +- cardinalities of values (number of distinct values) for every column and for every tuple of columns; +- conditional cardinalities: number of distinct values of one column under condition on value of another column; +- probability distributions of absolute value of integers; sign of signed integers; exponent and sign for floats; +- probability distributions of length of strings; +- probability of zero values of numbers; empty strings and arrays, NULLs; +- data compression ratio when compressed with LZ77 and entropy family of codecs; +- continuity (magnitude of difference) of time values across table; continuity of floating point values. +- date component of DateTime values; +- UTF-8 validity of string values; +- string values continue to look somewhat natural. + +Most of the properties above are viable for performance testing: + +reading data, filtering, aggregation and sorting will work at almost the same speed +as on original data due to saved cardinalities, magnitudes, compression ratios, etc. + +It works in deterministic fashion: you define a seed value and transform is totally determined by input data and by seed. +Some transforms are one to one and could be reversed, so you need to have large enough seed and keep it in secret. + +It use some cryptographic primitives to transform data, but from the cryptographic point of view, +It doesn't do anything properly and you should never consider the result as secure, unless you have other reasons for it. + +It may retain some data you don't want to publish. + +It always leave numbers 0, 1, -1 as is. Also it leaves dates, lengths of arrays and null flags exactly as in source data. +For example, you have a column IsMobile in your table with values 0 and 1. In transformed data, it will have the same value. +So, the user will be able to count exact ratio of mobile traffic. + +Another example, suppose you have some private data in your table, like user email and you don't want to publish any single email address. +If your table is large enough and contain multiple different emails and there is no email that have very high frequency than all others, +It will perfectly anonymize all data. But if you have small amount of different values in a column, it can possibly reproduce some of them. +And you should take care and look at exact algorithm, how this tool works, and probably fine tune some of it command line parameters. + +This tool works fine only with reasonable amount of data (at least 1000s of rows). diff --git a/programs/odbc-bridge/README.md b/docs/en/operations/utilities/odbc-bridge.md similarity index 100% rename from programs/odbc-bridge/README.md rename to docs/en/operations/utilities/odbc-bridge.md diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 9ed54eb9acd..be2f1ca5ba5 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -7,8 +7,9 @@ endif () # If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. option (ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse modes by default" ON) -option (ENABLE_CLICKHOUSE_SERVER ${ENABLE_CLICKHOUSE_ALL}) -option (ENABLE_CLICKHOUSE_CLIENT ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)" + ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL}) @@ -16,28 +17,30 @@ option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLIC # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ option (ENABLE_CLICKHOUSE_BENCHMARK "Queries benchmarking mode" ${ENABLE_CLICKHOUSE_ALL}) -# ??? -option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Configs processor (extract values etc.)" ${ENABLE_CLICKHOUSE_ALL}) -# ??? -option (ENABLE_CLICKHOUSE_COMPRESSOR ${ENABLE_CLICKHOUSE_ALL}) +# https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ +option (ENABLE_CLICKHOUSE_COMPRESSOR "Ddata compressor and decompressor" ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ option (ENABLE_CLICKHOUSE_COPIER "Inter-cluster data copying mode" ${ENABLE_CLICKHOUSE_ALL}) -# ??? -option (ENABLE_CLICKHOUSE_FORMAT ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_FORMAT "Queries pretty-printer and formatter with syntax highlighting" + ${ENABLE_CLICKHOUSE_ALL}) -# ??? -option (ENABLE_CLICKHOUSE_OBFUSCATOR ${ENABLE_CLICKHOUSE_ALL}) +# https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ +option (ENABLE_CLICKHOUSE_OBFUSCATOR "Table data obfuscator (convert real data to benchmark-ready one)" + ${ENABLE_CLICKHOUSE_ALL}) -# ??? -option (ENABLE_CLICKHOUSE_ODBC_BRIDGE ${ENABLE_CLICKHOUSE_ALL}) +# https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ +option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" + ${ENABLE_CLICKHOUSE_ALL}) if (CLICKHOUSE_SPLIT_BINARY) - option(ENABLE_CLICKHOUSE_INSTALL "???" OFF) + option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF) else () - option(ENABLE_CLICKHOUSE_INSTALL ${ENABLE_CLICKHOUSE_ALL}) + option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" + ${ENABLE_CLICKHOUSE_ALL}) endif () if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES)) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index aa947b22593..dcac329c845 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -672,7 +672,7 @@ int Server::main(const std::vector & /*args*/) /// /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h /// -#if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) +#if ENABLE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) /// Profilers cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) { @@ -696,7 +696,7 @@ int Server::main(const std::vector & /*args*/) /// Describe multiple reasons when query profiler cannot work. -#if !USE_UNWIND +#if !ENABLE_UNWIND LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); #endif diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 07e145359d8..3ed10261f5c 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -82,7 +82,7 @@ QueryProfilerBase::QueryProfilerBase(const UInt64 thread_id, const : log(&Poco::Logger::get("QueryProfiler")) , pause_signal(pause_signal_) { -#if USE_UNWIND +#if ENABLE_UNWIND /// Sanity check. if (!hasPHDRCache()) throw Exception("QueryProfiler cannot be used without PHDR cache, that is not available for TSan build", ErrorCodes::NOT_IMPLEMENTED); @@ -163,7 +163,7 @@ QueryProfilerBase::~QueryProfilerBase() template void QueryProfilerBase::tryCleanup() { -#if USE_UNWIND +#if ENABLE_UNWIND if (timer_id != nullptr && timer_delete(timer_id)) LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 8e2d09e0be2..54dc279a6b1 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -40,7 +40,7 @@ private: Poco::Logger * log; -#if USE_UNWIND +#if ENABLE_UNWIND /// Timer id from timer_create(2) timer_t timer_id = nullptr; #endif diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 677af269ca0..d3e5193aa6f 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -17,7 +17,7 @@ # include #endif -#if USE_UNWIND +#if ENABLE_UNWIND # include #endif @@ -287,7 +287,7 @@ StackTrace::StackTrace(NoCapture) void StackTrace::tryCapture() { size = 0; -#if USE_UNWIND +#if ENABLE_UNWIND size = unw_backtrace(frame_pointers.data(), capacity); __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); #endif diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index fd37e14685f..3a9c7edfab0 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -105,7 +105,7 @@ endif() target_link_libraries(clickhouse_functions PRIVATE hyperscan) -if(USE_SIMDJSON) +if(ENABLE_SIMDJSON) target_link_libraries(clickhouse_functions PRIVATE simdjson) endif() diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 40498353195..4da67a2d576 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -289,7 +289,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override { /// Choose JSONParser. -#if USE_SIMDJSON +#if ENABLE_SIMDJSON if (context.getSettingsRef().allow_simdjson) { FunctionJSONHelpers::Executor::run(block, arguments, result_pos, input_rows_count); diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a9adfa27e2c..4dc86184c81 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -4,7 +4,7 @@ # include "config_functions.h" #endif -#if USE_SIMDJSON +#if ENABLE_SIMDJSON # include # include # include diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 695b8efa3fb..894f01af16b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -43,7 +43,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) auto sink = std::make_shared(out); parquet::WriterProperties::Builder builder; -#if USE_SNAPPY +#if ENABLE_SNAPPY builder.compression(parquet::Compression::SNAPPY); #endif auto props = builder.build(); From e7b9319e120147735019f3a89a87b45087f86815 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 18 Sep 2020 16:27:50 +0300 Subject: [PATCH 137/390] If perf test definition changed, run everything + longer (as in master) Also some other perf test fixes --- docker/test/performance-comparison/compare.sh | 75 +++++++++++++------ .../test/performance-comparison/entrypoint.sh | 11 +-- docker/test/performance-comparison/report.py | 12 +-- tests/performance/columns_hashing.xml | 17 ++--- 4 files changed, 71 insertions(+), 44 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 8d7947b46a5..886dd0b74f6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -114,8 +114,6 @@ function run_tests # Just check that the script runs at all "$script_dir/perf.py" --help > /dev/null - changed_test_files="" - # Find the directory with test files. if [ -v CHPC_TEST_PATH ] then @@ -130,14 +128,6 @@ function run_tests else # For PRs, use newer test files so we can test these changes. test_prefix=right/performance - - # If only the perf tests were changed in the PR, we will run only these - # tests. The list of changed tests in changed-test.txt is prepared in - # entrypoint.sh from git diffs, because it has the cloned repo. Used - # to use rsync for that but it was really ugly and not always correct - # (e.g. when the reference SHA is really old and has some other - # differences to the tested SHA, besides the one introduced by the PR). - changed_test_files=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-tests.txt) fi # Determine which tests to run. @@ -146,19 +136,26 @@ function run_tests # Run only explicitly specified tests, if any. # shellcheck disable=SC2010 test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}") - elif [ "$changed_test_files" != "" ] + elif [ "$PR_TO_TEST" -ne 0 ] \ + && [ "$(wc -l < changed-test-definitions.txt)" -gt 0 ] \ + && [ "$(wc -l < changed-test-scripts.txt)" -eq 0 ] \ + && [ "$(wc -l < other-changed-files.txt)" -eq 0 ] then - # Use test files that changed in the PR. - test_files="$changed_test_files" + # If only the perf tests were changed in the PR, we will run only these + # tests. The lists of changed files are prepared in entrypoint.sh because + # it has the repository. + test_files=$(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-test-definitions.txt) else # The default -- run all tests found in the test dir. test_files=$(ls "$test_prefix"/*.xml) fi - # For PRs, test only a subset of queries, and run them less times. - # If the corresponding environment variables are already set, keep - # those values. - if [ "$PR_TO_TEST" == "0" ] + # For PRs w/o changes in test definitons and scripts, test only a subset of + # queries, and run them less times. If the corresponding environment variables + # are already set, keep those values. + if [ "$PR_TO_TEST" -ne 0 ] \ + && [ "$(wc -l < changed-test-definitions.txt)" -eq 0 ] \ + && [ "$(wc -l < changed-test-files.txt)" -eq 0 ] then CHPC_RUNS=${CHPC_RUNS:-13} CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-0} @@ -662,6 +659,38 @@ create table test_time engine Memory as from total_client_time_per_query full join queries using (test, query_index) group by test; +create view query_runs as select * from file('analyze/query-runs.tsv', TSV, + 'test text, query_index int, query_id text, version UInt8, time float'); + +-- +-- Guess the number of query runs used for this test. The number is required to +-- calculate and check the average query run time in the report. +-- We have to be careful, because we will encounter: +-- 1) partial queries which run only on one server +-- 2) short queries which run for a much higher number of times +-- 3) some errors that make query run for a different number of times on a +-- particular server. +-- +create view test_runs as + select test, + -- Default to 7 runs if there are only 'short' queries in the test, and + -- we can't determine the number of runs. + if((ceil(medianOrDefaultIf(t.runs, not short), 0) as r) != 0, r, 7) runs + from ( + select + -- The query id is the same for both servers, so no need to divide here. + uniqExact(query_id) runs, + (test, query_index) in + (select * from file('analyze/marked-short-queries.tsv', TSV, + 'test text, query_index int')) + as short, + test, query_index + from query_runs + group by test, query_index + ) t + group by test + ; + create table test_times_report engine File(TSV, 'report/test-times.tsv') as select wall_clock_time_per_test.test, real, toDecimal64(total_client_time, 3), @@ -669,11 +698,15 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv') as short_queries, toDecimal64(query_max, 3), toDecimal64(real / queries, 3) avg_real_per_query, - toDecimal64(query_min, 3) + toDecimal64(query_min, 3), + runs from test_time - -- wall clock times are also measured for skipped tests, so don't - -- do full join - left join wall_clock_time_per_test using test + -- wall clock times are also measured for skipped tests, so don't + -- do full join + left join wall_clock_time_per_test + on wall_clock_time_per_test.test = test_time.test + full join test_runs + on test_runs.test = test_time.test order by avg_real_per_query desc; -- report for all queries page, only main metric diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 9e9a46a3ce6..ed2e542eadd 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -97,13 +97,10 @@ then # tests for use by compare.sh. Compare to merge base, because master might be # far in the future and have unrelated test changes. base=$(git -C right/ch merge-base pr origin/master) - git -C right/ch diff --name-only "$base" pr | tee changed-tests.txt - if grep -vq '^tests/performance' changed-tests.txt - then - # Have some other changes besides the tests, so truncate the test list, - # meaning, run all tests. - : > changed-tests.txt - fi + git -C right/ch diff --name-only "$base" pr -- . | tee all-changed-files.txt + git -C right/ch diff --name-only "$base" pr -- tests/performance | tee changed-test-definitions.txt + git -C right/ch diff --name-only "$base" pr -- docker/test/performance-comparison | tee changed-test-scripts.txt + git -C right/ch diff --name-only "$base" pr -- :!tests/performance :!docker/test/performance-comparison | tee other-changed-files.txt fi # Set python output encoding so that we can print queries with Russian letters. diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index e9e2ac68c1e..ee67c3c0457 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -457,25 +457,25 @@ if args.report == 'main': return columns = [ - 'Test', #0 + 'Test', #0 'Wall clock time, s', #1 'Total client time, s', #2 - 'Total queries', #3 - 'Ignored short queries', #4 + 'Total queries', #3 + 'Ignored short queries', #4 'Longest query
(sum for all runs), s', #5 'Avg wall clock time
(sum for all runs), s', #6 'Shortest query
(sum for all runs), s', #7 + # 'Runs' #8 ] text = tableStart('Test times') text += tableHeader(columns) - nominal_runs = 7 # FIXME pass this as an argument - total_runs = (nominal_runs + 1) * 2 # one prewarm run, two servers - allowed_average_run_time = allowed_single_run_time + 60 / total_runs; # some allowance for fill/create queries + allowed_average_run_time = 3.75 # 60 seconds per test at 7 runs attrs = ['' for c in columns] for r in rows: anchor = f'{currentTableAnchor()}.{r[0]}' + total_runs = (int(r[8]) + 1) * 2 # one prewarm run, two servers if float(r[6]) > allowed_average_run_time * total_runs: # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index fb340c20ccd..271fff6e543 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -1,15 +1,12 @@ - - columns_hashing - - - test.hits + hits_10m_single + hits_100m_single - - - - - + select sum(UserID + 1 in (select UserID from hits_100m_single)) from hits_100m_single + select sum((UserID + 1, RegionID) in (select UserID, RegionID from hits_10m_single)) from hits_10m_single + select sum(URL in (select URL from hits_10m_single where URL != '')) from hits_10m_single + select sum(MobilePhoneModel in (select MobilePhoneModel from hits_10m_single where MobilePhoneModel != '')) from hits_10m_single + select sum((MobilePhoneModel, UserID + 1) in (select MobilePhoneModel, UserID from hits_100m_single where MobilePhoneModel != '')) from hits_100m_single From f6bce8bd1ea97a18cd8c05b4f4cea5bb7b84f8fa Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 18 Sep 2020 17:18:59 +0300 Subject: [PATCH 138/390] reverted USE to ENABLE --- base/daemon/CMakeLists.txt | 2 +- base/daemon/SentryWriter.cpp | 2 +- cmake/find/cxx.cmake | 6 +-- cmake/find/llvm.cmake | 2 +- cmake/find/sentry.cmake | 8 ++-- cmake/find/simdjson.cmake | 6 +-- cmake/find/snappy.cmake | 6 +-- cmake/find/unwind.cmake | 4 +- cmake/target.cmake | 2 +- cmake_flags_and_output.md | 41 ++++++++++++++++--- cmake_flags_generator.py | 2 +- programs/server/Server.cpp | 4 +- src/Common/QueryProfiler.cpp | 2 +- src/Common/QueryProfiler.h | 2 +- src/Common/StackTrace.cpp | 4 +- src/Functions/CMakeLists.txt | 2 +- src/Functions/FunctionsJSON.h | 2 +- src/Functions/SimdJSONParser.h | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- 19 files changed, 66 insertions(+), 35 deletions(-) diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 97278c593c8..26d59a57e7f 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -7,6 +7,6 @@ add_library (daemon target_include_directories (daemon PUBLIC ..) target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) -if (ENABLE_SENTRY) +if (USE_SENTRY) target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) endif () diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 7794be36c73..33f2b237dd5 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -16,7 +16,7 @@ # include #endif -#if ENABLE_SENTRY +#if USE_SENTRY # include // Y_IGNORE # include diff --git a/cmake/find/cxx.cmake b/cmake/find/cxx.cmake index 04bc3caf46a..b1da125e219 100644 --- a/cmake/find/cxx.cmake +++ b/cmake/find/cxx.cmake @@ -1,8 +1,8 @@ -option (ENABLE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) +option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) -if (NOT ENABLE_LIBCXX) +if (NOT USE_LIBCXX) if (USE_INTERNAL_LIBCXX_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with ENABLE_LIBCXX=OFF") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with USE_LIBCXX=OFF") endif() target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index e99d0f05c54..70d0e356c39 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -44,7 +44,7 @@ if (NOT USE_INTERNAL_LLVM_LIBRARY) set (USE_EMBEDDED_COMPILER 0) endif() - if (LLVM_FOUND AND OS_LINUX AND ENABLE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX) + if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX) message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages " "in Linux is incompatible with libc++ ABI. LLVM Will be disabled. Force: -DFORCE_LLVM_WITH_LIBCXX=ON") message (${RECONFIGURE_MESSAGE_LEVEL} "Unsupported LLVM configuration, cannot enable LLVM") diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 9739023a148..a986599abce 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -3,23 +3,23 @@ set (SENTRY_LIBRARY "sentry") set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include") if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive") - if (ENABLE_SENTRY) + if (USE_SENTRY) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library") endif() return() endif () if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_DARWIN AND COMPILER_CLANG)) - option (ENABLE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) + option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) set (SENTRY_TRANSPORT "curl" CACHE STRING "") set (SENTRY_BACKEND "none" CACHE STRING "") set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") set (SENTRY_PIC OFF CACHE BOOL "") set (BUILD_SHARED_LIBS OFF) - message (STATUS "Using sentry=${ENABLE_SENTRY}: ${SENTRY_LIBRARY}") + message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") -elseif (ENABLE_SENTRY) +elseif (USE_SENTRY) message (${RECONFIGURE_MESSAGE_LEVEL} "Sentry is not supported in current configuration") endif () diff --git a/cmake/find/simdjson.cmake b/cmake/find/simdjson.cmake index f1946e19a28..cffe20bdb2d 100644 --- a/cmake/find/simdjson.cmake +++ b/cmake/find/simdjson.cmake @@ -1,11 +1,11 @@ -option (ENABLE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) +option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h") message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init --recursive") - if (ENABLE_SIMDJSON) + if (USE_SIMDJSON) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library") endif() return() endif () -message(STATUS "Using simdjson=${ENABLE_SIMDJSON}") +message(STATUS "Using simdjson=${USE_SIMDJSON}") diff --git a/cmake/find/snappy.cmake b/cmake/find/snappy.cmake index 830d54975ef..2e1c8473904 100644 --- a/cmake/find/snappy.cmake +++ b/cmake/find/snappy.cmake @@ -1,8 +1,8 @@ -option(ENABLE_SNAPPY "Enable snappy library" ${ENABLE_LIBRARIES}) +option(USE_SNAPPY "Enable snappy library" ${ENABLE_LIBRARIES}) -if(NOT ENABLE_SNAPPY) +if(NOT USE_SNAPPY) if (USE_INTERNAL_SNAPPY_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with ENABLE_SNAPPY=OFF") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with USE_SNAPPY=OFF") endif() return() endif() diff --git a/cmake/find/unwind.cmake b/cmake/find/unwind.cmake index 3f8a0af078c..c9f5f30a5d6 100644 --- a/cmake/find/unwind.cmake +++ b/cmake/find/unwind.cmake @@ -1,6 +1,6 @@ -option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) +option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) -if (ENABLE_UNWIND) +if (USE_UNWIND) add_subdirectory(contrib/libunwind-cmake) set (UNWIND_LIBRARIES unwind) set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) diff --git a/cmake/target.cmake b/cmake/target.cmake index 832b9db2e4a..7174ca3c2a9 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -20,7 +20,7 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") # no protobuf -> no grpc - set (ENABLE_SNAPPY OFF CACHE INTERNAL "") + set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") # no snappy and protobuf -> no parquet set (ENABLE_ORC OFF CACHE INTERNAL "") # no arrow (parquet) -> no orc diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index 31614d77a72..cb444517ac8 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -71,7 +71,6 @@ cmake .. \ | [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | | [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | | [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | | [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | | [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | @@ -82,12 +81,44 @@ cmake .. \ | [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | | [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`ENABLE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`ENABLE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | | [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | | [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -| [`ENABLE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | +| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | +| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | +| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | +| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | +| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | +| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | +| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | +| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | +| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | +| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | +| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | +| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | +| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | +| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | +| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | +| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | +| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | +| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | ### External libraries system/bundled mode | Name | Default value | Description | Comment | diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 2c79595b564..96fab878f56 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -116,7 +116,7 @@ def process() -> None: f.write("### External libraries\n" + table_header) for k in sorted_keys: - if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): + if (k.startswith("ENABLE_") or k.startswith("USE_")) and entities[k][0].startswith("cmake"): f.write(entities[k][1] + "\n") ignored_keys.append(k) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index dcac329c845..aa947b22593 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -672,7 +672,7 @@ int Server::main(const std::vector & /*args*/) /// /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h /// -#if ENABLE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) +#if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) /// Profilers cannot work reliably with any other libunwind or without PHDR cache. if (hasPHDRCache()) { @@ -696,7 +696,7 @@ int Server::main(const std::vector & /*args*/) /// Describe multiple reasons when query profiler cannot work. -#if !ENABLE_UNWIND +#if !USE_UNWIND LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library."); #endif diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 3ed10261f5c..6f98cd40bc6 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -163,7 +163,7 @@ QueryProfilerBase::~QueryProfilerBase() template void QueryProfilerBase::tryCleanup() { -#if ENABLE_UNWIND +#if USE_UNWIND if (timer_id != nullptr && timer_delete(timer_id)) LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER)); diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 54dc279a6b1..8e2d09e0be2 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -40,7 +40,7 @@ private: Poco::Logger * log; -#if ENABLE_UNWIND +#if USE_UNWIND /// Timer id from timer_create(2) timer_t timer_id = nullptr; #endif diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index d3e5193aa6f..677af269ca0 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -17,7 +17,7 @@ # include #endif -#if ENABLE_UNWIND +#if USE_UNWIND # include #endif @@ -287,7 +287,7 @@ StackTrace::StackTrace(NoCapture) void StackTrace::tryCapture() { size = 0; -#if ENABLE_UNWIND +#if USE_UNWIND size = unw_backtrace(frame_pointers.data(), capacity); __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); #endif diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 3a9c7edfab0..fd37e14685f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -105,7 +105,7 @@ endif() target_link_libraries(clickhouse_functions PRIVATE hyperscan) -if(ENABLE_SIMDJSON) +if(USE_SIMDJSON) target_link_libraries(clickhouse_functions PRIVATE simdjson) endif() diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 4da67a2d576..40498353195 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -289,7 +289,7 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override { /// Choose JSONParser. -#if ENABLE_SIMDJSON +#if USE_SIMDJSON if (context.getSettingsRef().allow_simdjson) { FunctionJSONHelpers::Executor::run(block, arguments, result_pos, input_rows_count); diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index 4dc86184c81..a9adfa27e2c 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -4,7 +4,7 @@ # include "config_functions.h" #endif -#if ENABLE_SIMDJSON +#if USE_SIMDJSON # include # include # include diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 894f01af16b..695b8efa3fb 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -43,7 +43,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) auto sink = std::make_shared(out); parquet::WriterProperties::Builder builder; -#if ENABLE_SNAPPY +#if USE_SNAPPY builder.compression(parquet::Compression::SNAPPY); #endif auto props = builder.build(); From 0f5f95930f5d14276481f28814ac15f5fc6fd0df Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 18 Sep 2020 17:21:10 +0300 Subject: [PATCH 139/390] fixed duplicates --- cmake_flags_and_output.md | 41 +++++---------------------------------- cmake_flags_generator.py | 2 +- 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md index cb444517ac8..033cdf3e943 100644 --- a/cmake_flags_and_output.md +++ b/cmake_flags_and_output.md @@ -83,42 +83,6 @@ cmake .. \ | [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | | [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | | [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | -| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | -| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | -| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | -| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | -| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | -| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | -| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | -| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | -| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | -| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | -| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | -| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | -| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | -| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | ### External libraries system/bundled mode | Name | Default value | Description | Comment | @@ -180,7 +144,12 @@ cmake .. \ | [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | | [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | | [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | | [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | | [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | | [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | | [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | diff --git a/cmake_flags_generator.py b/cmake_flags_generator.py index 96fab878f56..2c79595b564 100755 --- a/cmake_flags_generator.py +++ b/cmake_flags_generator.py @@ -116,7 +116,7 @@ def process() -> None: f.write("### External libraries\n" + table_header) for k in sorted_keys: - if (k.startswith("ENABLE_") or k.startswith("USE_")) and entities[k][0].startswith("cmake"): + if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): f.write(entities[k][1] + "\n") ignored_keys.append(k) From 822a78fa585c9a9e5b342f9d17dc5e0490449e01 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Fri, 18 Sep 2020 17:25:45 +0300 Subject: [PATCH 140/390] call under mutex --- src/Dictionaries/CacheDictionary.cpp | 8 ++++---- src/Dictionaries/CacheDictionary.h | 10 ++++++---- src/Dictionaries/CacheDictionary.inc.h | 14 ++++---------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index 29aee9bfc21..86e8c9390ad 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -908,7 +908,7 @@ void CacheDictionary::update(UpdateUnitPtr & update_unit_ptr) const else cell.setExpiresAt(std::chrono::time_point::max()); - update_unit_ptr->getPresentIdHandler()(id, cell_idx); + update_unit_ptr->callPresentIdHandler(id, cell_idx); /// mark corresponding id as found remaining_ids[id] = 1; } @@ -970,9 +970,9 @@ void CacheDictionary::update(UpdateUnitPtr & update_unit_ptr) const if (was_default) cell.setDefault(); if (was_default) - update_unit_ptr->getAbsentIdHandler()(id, cell_idx); + update_unit_ptr->callAbsentIdHandler(id, cell_idx); else - update_unit_ptr->getPresentIdHandler()(id, cell_idx); + update_unit_ptr->callPresentIdHandler(id, cell_idx); continue; } /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`. @@ -1004,7 +1004,7 @@ void CacheDictionary::update(UpdateUnitPtr & update_unit_ptr) const setDefaultAttributeValue(attribute, cell_idx); /// inform caller that the cell has not been found - update_unit_ptr->getAbsentIdHandler()(id, cell_idx); + update_unit_ptr->callAbsentIdHandler(id, cell_idx); } ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num); diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 5e7e272ff2e..ee4229b3249 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -399,16 +399,18 @@ private: absent_id_handler([](Key, size_t){}){} - PresentIdHandler getPresentIdHandler() + void callPresentIdHandler(Key key, size_t cell_idx) { std::lock_guard lock(callback_mutex); - return can_use_callback ? present_id_handler : PresentIdHandler{}; + if (can_use_callback) + present_id_handler(key, cell_idx); } - AbsentIdHandler getAbsentIdHandler() + void callAbsentIdHandler(Key key, size_t cell_idx) { std::lock_guard lock(callback_mutex); - return can_use_callback ? absent_id_handler : AbsentIdHandler{}; + if (can_use_callback) + absent_id_handler(key, cell_idx); } std::vector requested_ids; diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 1f0bca0d01f..5aeed17e2c2 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -154,17 +154,11 @@ void CacheDictionary::getItemsNumberImpl( { const auto attribute_value = attribute_array[cell_idx]; - auto cache_not_found_it = cache_not_found_ids.find(id); + for (const size_t row : cache_not_found_ids[id]) + out[row] = static_cast(attribute_value); - if (cache_not_found_it != cache_not_found_ids.end()) - for (const size_t row : cache_not_found_it->second) - out[row] = static_cast(attribute_value); - - auto cache_expired_it = cache_expired_ids.find(id); - - if (cache_expired_it != cache_expired_ids.end()) - for (const size_t row : cache_expired_it->second) - out[row] = static_cast(attribute_value); + for (const size_t row : cache_expired_ids[id]) + out[row] = static_cast(attribute_value); }; auto on_id_not_found = [&] (auto, auto) {}; From 122e34cf19081952c3df8f37e3d639df1fdfdaa3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 17:38:28 +0300 Subject: [PATCH 141/390] Fix "Arcadia" --- src/Common/ya.make | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ya.make b/src/Common/ya.make index 4d558b4dbc9..72f1fa42756 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -87,7 +87,6 @@ SRCS( StatusFile.cpp StatusInfo.cpp Stopwatch.cpp - StringRef.cpp StringUtils/StringUtils.cpp StudentTTest.cpp SymbolIndex.cpp From 72c2201287f0e67af070c41c02361ad01bb9d5aa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 17:39:33 +0300 Subject: [PATCH 142/390] Fix "Arcadia" --- src/Common/ya.make | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ya.make b/src/Common/ya.make index 9e4f4cd76c0..9b671e7b9ee 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -88,7 +88,6 @@ SRCS( StatusFile.cpp StatusInfo.cpp Stopwatch.cpp - StringRef.cpp StringUtils/StringUtils.cpp StudentTTest.cpp SymbolIndex.cpp From 2530b71aa6cd1ede8e53930a6069db0e126f7364 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Fri, 18 Sep 2020 17:48:05 +0300 Subject: [PATCH 143/390] style --- src/Dictionaries/CacheDictionary.inc.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 5aeed17e2c2..8867d6a3c4a 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -148,9 +148,9 @@ void CacheDictionary::getItemsNumberImpl( std::begin(cache_expired_ids), std::end(cache_expired_ids), std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - auto on_cell_updated = - [&attribute_array, &cache_not_found_ids, &cache_expired_ids, &out] - (const auto id, const auto cell_idx) + auto on_cell_updated = + [&attribute_array, &cache_not_found_ids, &cache_expired_ids, &out] + (const auto id, const auto cell_idx) { const auto attribute_value = attribute_array[cell_idx]; From 0da19ab46de88f01cea34e7d7fcab057fd6ce2c4 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 18 Sep 2020 18:30:00 +0300 Subject: [PATCH 144/390] Add option to disable ttl move on data part insert --- src/Disks/IVolume.h | 6 ++- src/Disks/VolumeJBOD.cpp | 3 ++ src/Storages/MergeTree/MergeTreeData.cpp | 43 ++++++++++++++---- src/Storages/MergeTree/MergeTreeData.h | 12 ++--- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../config.d/storage_configuration.xml | 12 +++++ tests/integration/test_ttl_move/test.py | 45 +++++++++++++++++++ 7 files changed, 107 insertions(+), 16 deletions(-) diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index 5e7f09e1d04..0f38fe4d22e 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -36,10 +36,11 @@ using Volumes = std::vector; class IVolume : public Space { public: - IVolume(String name_, Disks disks_, size_t max_data_part_size_ = 0) + IVolume(String name_, Disks disks_, size_t max_data_part_size_ = 0, bool perform_ttl_move_on_insert_ = true) : disks(std::move(disks_)) , name(name_) , max_data_part_size(max_data_part_size_) + , perform_ttl_move_on_insert(perform_ttl_move_on_insert_) { } @@ -70,6 +71,9 @@ protected: public: /// Max size of reservation, zero means unlimited size UInt64 max_data_part_size = 0; + /// Should a new data part be synchronously moved to a volume according to ttl on insert + /// or move this part in background task asynchronously after insert. + bool perform_ttl_move_on_insert; }; /// Reservation for multiple disks at once. Can be used in RAID1 implementation. diff --git a/src/Disks/VolumeJBOD.cpp b/src/Disks/VolumeJBOD.cpp index bf9dcf7f5d8..3ac8a50acfb 100644 --- a/src/Disks/VolumeJBOD.cpp +++ b/src/Disks/VolumeJBOD.cpp @@ -53,6 +53,9 @@ VolumeJBOD::VolumeJBOD( static constexpr UInt64 MIN_PART_SIZE = 8u * 1024u * 1024u; if (max_data_part_size != 0 && max_data_part_size < MIN_PART_SIZE) LOG_WARNING(logger, "Volume {} max_data_part_size is too low ({} < {})", backQuote(name), ReadableSize(max_data_part_size), ReadableSize(MIN_PART_SIZE)); + + /// Default value is 'true' due to backward compatibility. + perform_ttl_move_on_insert = config.getBool(config_prefix + ".perform_ttl_move_on_insert", true); } DiskPtr VolumeJBOD::getDisk(size_t /* index */) const diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 5969ecc5baf..853ea8eabcf 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2961,11 +2961,12 @@ ReservationPtr MergeTreeData::tryReserveSpace(UInt64 expected_size, SpacePtr spa ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_size, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, - size_t min_volume_index) const + size_t min_volume_index, + bool is_insert) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); - ReservationPtr reservation = tryReserveSpacePreferringTTLRules(expected_size, ttl_infos, time_of_move, min_volume_index); + ReservationPtr reservation = tryReserveSpacePreferringTTLRules(expected_size, ttl_infos, time_of_move, min_volume_index, is_insert); return checkAndReturnReservation(expected_size, std::move(reservation)); } @@ -2973,7 +2974,8 @@ ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_siz ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_size, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, - size_t min_volume_index) const + size_t min_volume_index, + bool is_insert) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); @@ -2984,13 +2986,13 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (ttl_entry) { - SpacePtr destination_ptr = getDestinationForTTL(*ttl_entry); + SpacePtr destination_ptr = getDestinationForTTL(*ttl_entry, is_insert); if (!destination_ptr) { if (ttl_entry->destination_type == DataDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment", ttl_entry->destination_name, log_name); else if (ttl_entry->destination_type == DataDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found", ttl_entry->destination_name, log_name); + LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment", ttl_entry->destination_name, log_name); } else { @@ -3010,13 +3012,36 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; } -SpacePtr MergeTreeData::getDestinationForTTL(const TTLDescription & ttl) const +SpacePtr MergeTreeData::getDestinationForTTL(const TTLDescription & ttl, bool is_insert) const { auto policy = getStoragePolicy(); if (ttl.destination_type == DataDestinationType::VOLUME) - return policy->getVolumeByName(ttl.destination_name); + { + auto volume = policy->getVolumeByName(ttl.destination_name); + + if (!volume) + return {}; + + if (is_insert && !volume->perform_ttl_move_on_insert) + return {}; + + return volume; + } else if (ttl.destination_type == DataDestinationType::DISK) - return policy->getDiskByName(ttl.destination_name); + { + auto disk = policy->getDiskByName(ttl.destination_name); + if (!disk) + return {}; + + auto volume = policy->getVolume(policy->getVolumeIndexByDisk(disk)); + if (!volume) + return {}; + + if (is_insert && !volume->perform_ttl_move_on_insert) + return {}; + + return disk; + } else return {}; } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 82f118a4c0f..59628371ac8 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -624,13 +624,15 @@ public: UInt64 expected_size, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, - size_t min_volume_index = 0) const; + size_t min_volume_index = 0, + bool is_insert = false) const; ReservationPtr tryReserveSpacePreferringTTLRules( UInt64 expected_size, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, - size_t min_volume_index = 0) const; + size_t min_volume_index = 0, + bool is_insert = false) const; /// Choose disk with max available free space /// Reserves 0 bytes @@ -638,9 +640,9 @@ public: /// Return alter conversions for part which must be applied on fly. AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const; - /// Returns destination disk or volume for the TTL rule according to current - /// storage policy - SpacePtr getDestinationForTTL(const TTLDescription & ttl) const; + /// Returns destination disk or volume for the TTL rule according to current storage policy + /// 'is_insert' - is TTL move performed on new data part insert. + SpacePtr getDestinationForTTL(const TTLDescription & ttl, bool is_insert = false) const; /// Checks if given part already belongs destination disk or volume for the /// TTL rule. diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index e5b684a1361..b8de87ecd3a 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -237,7 +237,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); - ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr)); + ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr), true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); auto new_data_part = data.createPart( diff --git a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml index 47bf9f56cdd..e96bde89ca9 100644 --- a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -83,6 +83,18 @@ + + + +
+ jbod1 +
+ + external + false + + + diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index ad822bc6545..cd822025609 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1102,3 +1102,48 @@ limitations under the License.""" finally: node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + + +@pytest.mark.parametrize("name,dest_type,engine", [ + ("mt_test_disabled_ttl_move_on_insert_work", "DISK", "MergeTree()"), + ("mt_test_disabled_ttl_move_on_insert_work", "VOLUME", "MergeTree()"), + ("replicated_mt_test_disabled_ttl_move_on_insert_work", "DISK", "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')"), + ("replicated_mt_test_disabled_ttl_move_on_insert_work", "VOLUME", "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')"), +]) +def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): + try: + node1.query("SYSTEM STOP MOVES") + + node1.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO {dest_type} 'external' + SETTINGS storage_policy='jbod_without_instant_ttl_move' + """.format(name=name, dest_type=dest_type, engine=engine)) + + data = [] # 10MB in total + for i in range(10): + data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format( + time.time() - 1))) # 1MB row + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == "jbod1" + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + node1.query("SYSTEM START MOVES") + time.sleep(3) + + used_disks = get_used_disks_for_table(node1, name) + assert set(used_disks) == "external" + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + try: + node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + except: + pass \ No newline at end of file From 6b4b5898a79c494a0607211156687197b82d8ffc Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 18 Sep 2020 18:41:14 +0300 Subject: [PATCH 145/390] Renaming. --- src/Storages/MergeTree/MergeTreeData.cpp | 38 ++++++++++--------- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 4 +- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 853ea8eabcf..6d7af9e88e4 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -511,7 +511,7 @@ void MergeTreeData::checkTTLExpressions(const StorageInMemoryMetadata & new_meta { for (const auto & move_ttl : new_table_ttl.move_ttl) { - if (!getDestinationForTTL(move_ttl)) + if (!getDestinationForMoveTTL(move_ttl)) { String message; if (move_ttl.destination_type == DataDestinationType::DISK) @@ -2982,17 +2982,19 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ auto metadata_snapshot = getInMemoryMetadataPtr(); ReservationPtr reservation; - auto ttl_entry = selectTTLDescriptionForTTLInfos(metadata_snapshot->getMoveTTLs(), ttl_infos.moves_ttl, time_of_move, true); + auto move_ttl_entry = selectTTLDescriptionForTTLInfos(metadata_snapshot->getMoveTTLs(), ttl_infos.moves_ttl, time_of_move, true); - if (ttl_entry) + if (move_ttl_entry) { - SpacePtr destination_ptr = getDestinationForTTL(*ttl_entry, is_insert); + SpacePtr destination_ptr = getDestinationForMoveTTL(*move_ttl_entry, is_insert); if (!destination_ptr) { - if (ttl_entry->destination_type == DataDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment", ttl_entry->destination_name, log_name); - else if (ttl_entry->destination_type == DataDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment", ttl_entry->destination_name, log_name); + if (move_ttl_entry->destination_type == DataDestinationType::VOLUME) + LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment", + move_ttl_entry->destination_name, log_name); + else if (move_ttl_entry->destination_type == DataDestinationType::DISK) + LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment", + move_ttl_entry->destination_name, log_name); } else { @@ -3000,10 +3002,12 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ if (reservation) return reservation; else - if (ttl_entry->destination_type == DataDestinationType::VOLUME) - LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); - else if (ttl_entry->destination_type == DataDestinationType::DISK) - LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but there is not enough space", ttl_entry->destination_name, log_name); + if (move_ttl_entry->destination_type == DataDestinationType::VOLUME) + LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but there is not enough space", + move_ttl_entry->destination_name, log_name); + else if (move_ttl_entry->destination_type == DataDestinationType::DISK) + LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but there is not enough space", + move_ttl_entry->destination_name, log_name); } } @@ -3012,12 +3016,12 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_ return reservation; } -SpacePtr MergeTreeData::getDestinationForTTL(const TTLDescription & ttl, bool is_insert) const +SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl, bool is_insert) const { auto policy = getStoragePolicy(); - if (ttl.destination_type == DataDestinationType::VOLUME) + if (move_ttl.destination_type == DataDestinationType::VOLUME) { - auto volume = policy->getVolumeByName(ttl.destination_name); + auto volume = policy->getVolumeByName(move_ttl.destination_name); if (!volume) return {}; @@ -3027,9 +3031,9 @@ SpacePtr MergeTreeData::getDestinationForTTL(const TTLDescription & ttl, bool is return volume; } - else if (ttl.destination_type == DataDestinationType::DISK) + else if (move_ttl.destination_type == DataDestinationType::DISK) { - auto disk = policy->getDiskByName(ttl.destination_name); + auto disk = policy->getDiskByName(move_ttl.destination_name); if (!disk) return {}; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 59628371ac8..0bd8dd9bc0c 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -642,7 +642,7 @@ public: AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const; /// Returns destination disk or volume for the TTL rule according to current storage policy /// 'is_insert' - is TTL move performed on new data part insert. - SpacePtr getDestinationForTTL(const TTLDescription & ttl, bool is_insert = false) const; + SpacePtr getDestinationForMoveTTL(const TTLDescription & move_ttl, bool is_insert = false) const; /// Checks if given part already belongs destination disk or volume for the /// TTL rule. diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 586c4393dfb..c5c6a63abc4 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -136,9 +136,9 @@ bool MergeTreePartsMover::selectPartsForMove( ReservationPtr reservation; if (ttl_entry) { - auto destination = data->getDestinationForTTL(*ttl_entry); + auto destination = data->getDestinationForMoveTTL(*ttl_entry); if (destination && !data->isPartInTTLDestination(*ttl_entry, *part)) - reservation = data->tryReserveSpace(part->getBytesOnDisk(), data->getDestinationForTTL(*ttl_entry)); + reservation = data->tryReserveSpace(part->getBytesOnDisk(), data->getDestinationForMoveTTL(*ttl_entry)); } if (reservation) /// Found reservation by TTL rule. From 28afbafa08c2362a121334294082ba5599c6b753 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Fri, 18 Sep 2020 19:25:20 +0300 Subject: [PATCH 146/390] fix crash in RIGHT or FULL JOIN switch --- src/Interpreters/ExpressionAnalyzer.cpp | 4 +-- src/Interpreters/ExpressionAnalyzer.h | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 5 +-- src/Interpreters/JoinSwitcher.h | 35 +++++++++++++++++++ .../01476_right_full_join_switch.reference | 9 +++++ .../01476_right_full_join_switch.sql | 26 ++++++++++++++ 6 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01476_right_full_join_switch.reference create mode 100644 tests/queries/0_stateless/01476_right_full_join_switch.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index d9fc44d9b8f..8d67672612c 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1142,8 +1142,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( if (hasJoin()) { /// You may find it strange but we support read_in_order for HashJoin and do not support for MergeJoin. - bool has_delayed_stream = query_analyzer.analyzedJoin().needStreamWithNonJoinedRows(); - join_allow_read_in_order = typeid_cast(join.get()) && !has_delayed_stream; + join_has_delayed_stream = query_analyzer.analyzedJoin().needStreamWithNonJoinedRows(); + join_allow_read_in_order = typeid_cast(join.get()) && !join_has_delayed_stream; } optimize_read_in_order = diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 1cc9d75b19f..cbfebafa439 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -176,6 +176,7 @@ struct ExpressionAnalysisResult bool remove_where_filter = false; bool optimize_read_in_order = false; bool optimize_aggregation_in_order = false; + bool join_has_delayed_stream = false; ExpressionActionsPtr before_array_join; ArrayJoinActionPtr array_join; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index cd2c16813b4..f049376156b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include @@ -925,8 +925,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu join_step->setStepDescription("JOIN"); query_plan.addStep(std::move(join_step)); - if (auto stream = join->createStreamWithNonJoinedRows(join_result_sample, settings.max_block_size)) + if (expressions.join_has_delayed_stream) { + auto stream = std::make_shared(*join, join_result_sample, settings.max_block_size); auto source = std::make_shared(std::move(stream)); auto add_non_joined_rows_step = std::make_unique( query_plan.getCurrentDataStream(), std::move(source)); diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h index ea9b94546ef..23f5dff54d7 100644 --- a/src/Interpreters/JoinSwitcher.h +++ b/src/Interpreters/JoinSwitcher.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -75,4 +76,38 @@ private: void switchJoin(); }; + +/// Creates NonJoinedBlockInputStream on the first read. Allows to swap join algo before it. +class LazyNonJoinedBlockInputStream : public IBlockInputStream +{ +public: + LazyNonJoinedBlockInputStream(const IJoin & join_, const Block & block, UInt64 max_block_size_) + : join(join_) + , result_sample_block(block) + , max_block_size(max_block_size_) + {} + + String getName() const override { return "LazyNonMergeJoined"; } + Block getHeader() const override { return result_sample_block; } + +protected: + Block readImpl() override + { + if (!stream) + { + stream = join.createStreamWithNonJoinedRows(result_sample_block, max_block_size); + if (!stream) + return {}; + } + + return stream->read(); + } + +private: + BlockInputStreamPtr stream; + const IJoin & join; + Block result_sample_block; + UInt64 max_block_size; +}; + } diff --git a/tests/queries/0_stateless/01476_right_full_join_switch.reference b/tests/queries/0_stateless/01476_right_full_join_switch.reference new file mode 100644 index 00000000000..1f839b86013 --- /dev/null +++ b/tests/queries/0_stateless/01476_right_full_join_switch.reference @@ -0,0 +1,9 @@ +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +- +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) diff --git a/tests/queries/0_stateless/01476_right_full_join_switch.sql b/tests/queries/0_stateless/01476_right_full_join_switch.sql new file mode 100644 index 00000000000..5d041843ee2 --- /dev/null +++ b/tests/queries/0_stateless/01476_right_full_join_switch.sql @@ -0,0 +1,26 @@ +SET join_algorithm = 'auto'; +SET max_bytes_in_join = 100; + +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS nr; + +CREATE TABLE t (`x` UInt32, `s` LowCardinality(String)) ENGINE = Memory; +CREATE TABLE nr (`x` Nullable(UInt32), `s` Nullable(String)) ENGINE = Memory; + +INSERT INTO t VALUES (1, 'l'); +INSERT INTO nr VALUES (2, NULL); + +SET join_use_nulls = 0; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (x) ORDER BY t.x; + +DROP TABLE t; +DROP TABLE nr; From da04a130ed5a33501664df764e319e52673cfbea Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 18 Sep 2020 20:45:30 +0300 Subject: [PATCH 147/390] Add option to disable ttl move on data part insert - minor fixes. --- src/Disks/IVolume.h | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 2 +- .../configs/config.d/storage_configuration.xml | 1 + tests/integration/test_ttl_move/test.py | 8 +++++--- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index 0f38fe4d22e..eaf3bf1dbd4 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -73,7 +73,7 @@ public: UInt64 max_data_part_size = 0; /// Should a new data part be synchronously moved to a volume according to ttl on insert /// or move this part in background task asynchronously after insert. - bool perform_ttl_move_on_insert; + bool perform_ttl_move_on_insert = true; }; /// Reservation for multiple disks at once. Can be used in RAID1 implementation. diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 0e42d267729..f9fb157942a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -276,7 +276,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( ReadBufferFromString ttl_infos_buffer(ttl_infos_string); assertString("ttl format version: 1\n", ttl_infos_buffer); ttl_infos.read(ttl_infos_buffer); - reservation = data.reserveSpacePreferringTTLRules(sum_files_size, ttl_infos, std::time(nullptr)); + reservation = data.reserveSpacePreferringTTLRules(sum_files_size, ttl_infos, std::time(nullptr), 0, true); } else reservation = data.reserveSpace(sum_files_size); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index b8de87ecd3a..739aff31a06 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -237,7 +237,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); - ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr), true); + ReservationPtr reservation = data.reserveSpacePreferringTTLRules(expected_size, move_ttl_infos, time(nullptr), 0, true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); auto new_data_part = data.createPart( diff --git a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml index e96bde89ca9..98cd8160ac6 100644 --- a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -88,6 +88,7 @@
jbod1 + jbod2
external diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index cd822025609..990b2a742b4 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1113,6 +1113,7 @@ limitations under the License.""" def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): try: node1.query("SYSTEM STOP MOVES") + node2.query("SYSTEM STOP MOVES") node1.query(""" CREATE TABLE {name} ( @@ -1132,18 +1133,19 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) used_disks = get_used_disks_for_table(node1, name) - assert set(used_disks) == "jbod1" + assert set(used_disks) == {"jbod1"} assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" node1.query("SYSTEM START MOVES") + node2.query("SYSTEM START MOVES") time.sleep(3) used_disks = get_used_disks_for_table(node1, name) - assert set(used_disks) == "external" + assert set(used_disks) == {"external"} assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" finally: try: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) except: - pass \ No newline at end of file + pass From 800ad407e86287cc8144d0ba8dda72b672e26295 Mon Sep 17 00:00:00 2001 From: myrrc Date: Fri, 18 Sep 2020 20:49:59 +0300 Subject: [PATCH 148/390] fix --- src/Common/QueryProfiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 6f98cd40bc6..07e145359d8 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -82,7 +82,7 @@ QueryProfilerBase::QueryProfilerBase(const UInt64 thread_id, const : log(&Poco::Logger::get("QueryProfiler")) , pause_signal(pause_signal_) { -#if ENABLE_UNWIND +#if USE_UNWIND /// Sanity check. if (!hasPHDRCache()) throw Exception("QueryProfiler cannot be used without PHDR cache, that is not available for TSan build", ErrorCodes::NOT_IMPLEMENTED); From d842cb704fb6736620e1a8819d00e5cc1d3d3320 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 18 Sep 2020 23:50:28 +0800 Subject: [PATCH 149/390] Allow mutations to work with key constraints. --- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 3 +++ .../test_mutations_with_merge_tree/configs/users.xml | 2 ++ tests/integration/test_mutations_with_merge_tree/test.py | 8 ++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 89631b713ed..48309303f59 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1079,6 +1079,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto context_for_reading = context; context_for_reading.setSetting("max_streams_to_max_threads_ratio", 1); context_for_reading.setSetting("max_threads", 1); + /// Allow mutations to work when force_index_by_date or force_primary_key is on. + context_for_reading.setSetting("force_index_by_date", 0); + context_for_reading.setSetting("force_primary_key", 0); MutationCommands commands_for_part; for (const auto & command : commands) diff --git a/tests/integration/test_mutations_with_merge_tree/configs/users.xml b/tests/integration/test_mutations_with_merge_tree/configs/users.xml index e1dd7fb0638..47dea62971b 100644 --- a/tests/integration/test_mutations_with_merge_tree/configs/users.xml +++ b/tests/integration/test_mutations_with_merge_tree/configs/users.xml @@ -3,6 +3,8 @@ 500 + 1 + 1 diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index 019f8c2ea40..25bc0df8e7c 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -16,7 +16,7 @@ def started_cluster(): instance_test_mutations.query( '''CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)''') instance_test_mutations.query( - '''INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1)''') + '''INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1) SETTINGS force_index_by_date = 0, force_primary_key = 0''') yield cluster finally: cluster.shutdown() @@ -38,14 +38,14 @@ def test_mutations_with_merge_background_task(started_cluster): instance_test_mutations.query('''DETACH TABLE test_mutations_with_ast_elements''') instance_test_mutations.query('''ATTACH TABLE test_mutations_with_ast_elements''') return int(instance.query( - "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip()) + "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0").rstrip()) if get_done_mutations(instance_test_mutations) == 100: all_done = True break print instance_test_mutations.query( - "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' FORMAT TSVWithNames") + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames") assert all_done @@ -59,4 +59,4 @@ def test_mutations_with_truncate_table(started_cluster): instance_test_mutations.query("TRUNCATE TABLE test_mutations_with_ast_elements") assert instance_test_mutations.query( - "SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements'").rstrip() == '0' + "SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements SETTINGS force_index_by_date = 0, force_primary_key = 0'").rstrip() == '0' From 6e6909f47d43b5d14b7550c519dba125b844c00a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 21:54:54 +0300 Subject: [PATCH 150/390] Fix "Arcadia" --- src/Common/getMappedArea.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp index 018db16038b..921d95ab740 100644 --- a/src/Common/getMappedArea.cpp +++ b/src/Common/getMappedArea.cpp @@ -15,7 +15,6 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; } @@ -69,7 +68,12 @@ std::pair getMappedArea(void * ptr) namespace DB { -std::pair getMappedArea(void * ptr) +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +std::pair getMappedArea(void *) { throw Exception("The function getMappedArea is implemented only for Linux", ErrorCodes::NOT_IMPLEMENTED); } From 8a8e4f8a41e59e4ce73b823cdf721af02dce92d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 13:28:54 +0300 Subject: [PATCH 151/390] Attempt to speed-up clang build --- src/Common/PODArray.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 1084f0800cc..7bd9550500e 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -214,6 +214,9 @@ public: void clear() { c_end = c_start; } template +#if defined(__clang__) + ALWAYS_INLINE /// Better performance in clang build, worse performance in gcc build. +#endif void reserve(size_t n, TAllocatorParams &&... allocator_params) { if (n > capacity()) From 77be35a2b83dd1427460aee63767346b4e2cee49 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 18 Sep 2020 21:59:56 +0300 Subject: [PATCH 152/390] Fixed test_disabled_ttl_move_on_insert test --- .../configs/config.d/storage_configuration.xml | 1 - tests/integration/test_ttl_move/test.py | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml index 98cd8160ac6..e96bde89ca9 100644 --- a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -88,7 +88,6 @@
jbod1 - jbod2
external diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 990b2a742b4..5c97e097f1c 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1112,9 +1112,6 @@ limitations under the License.""" ]) def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): try: - node1.query("SYSTEM STOP MOVES") - node2.query("SYSTEM STOP MOVES") - node1.query(""" CREATE TABLE {name} ( s1 String, @@ -1125,6 +1122,8 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): SETTINGS storage_policy='jbod_without_instant_ttl_move' """.format(name=name, dest_type=dest_type, engine=engine)) + node1.query("SYSTEM STOP MOVES {}".format(name)) + data = [] # 10MB in total for i in range(10): data.append(("'{}'".format(get_random_string(1024 * 1024)), "toDateTime({})".format( @@ -1136,8 +1135,7 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): assert set(used_disks) == {"jbod1"} assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" - node1.query("SYSTEM START MOVES") - node2.query("SYSTEM START MOVES") + node1.query("SYSTEM START MOVES {}").format(name) time.sleep(3) used_disks = get_used_disks_for_table(node1, name) From 1f429b3d5298e146b51ca77305b3a43786ae0e52 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 17 Sep 2020 13:39:13 +0300 Subject: [PATCH 153/390] Set enable tests to zero for performance build --- docker/packager/packager | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/packager b/docker/packager/packager index 909f20acd6d..0a14102ec04 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -105,6 +105,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ # Create combined output archive for split build and for performance tests. if package_type == "performance": result.append("COMBINED_OUTPUT=performance") + cmake_flags.append("-DENABLE_TESTS=0") elif split_binary: result.append("COMBINED_OUTPUT=shared_build") From 368ddada33d19b547bf7bcdbc22a2d057aedb8a5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 17 Sep 2020 17:01:48 +0300 Subject: [PATCH 154/390] Fix binary build --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 72adba5d762..dc34e7297dc 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -20,7 +20,7 @@ rm -f CMakeCache.txt cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS .. ninja $NINJA_FLAGS clickhouse-bundle mv ./programs/clickhouse* /output -mv ./src/unit_tests_dbms /output +mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds find . -name '*.so' -print -exec mv '{}' /output \; find . -name '*.so.*' -print -exec mv '{}' /output \; From e1ffa07a3983781009da90f208841cc815a96fc4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 22:08:53 +0300 Subject: [PATCH 155/390] Resolve review comments --- src/Storages/StorageDistributed.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 073dd7cc754..e573b264e4f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -876,10 +876,9 @@ void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) { for (const DiskPtr & disk : data_volume->getDisks()) { - const String path(disk->getPath()); - auto new_path = path + new_path_to_table_data; - disk->moveDirectory(path + relative_data_path, new_path); + disk->moveDirectory(relative_data_path, new_path_to_table_data); + auto new_path = disk->getPath() + new_path_to_table_data; LOG_DEBUG(log, "Updating path to {}", new_path); std::lock_guard lock(cluster_nodes_mutex); From 1fc3aa3ea8ae87d0a184bd32abeb690dd4885ef5 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 18 Sep 2020 22:10:49 +0300 Subject: [PATCH 156/390] Fixed test_disabled_ttl_move_on_insert test --- tests/integration/test_ttl_move/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 5c97e097f1c..377ee0e5d75 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1135,7 +1135,7 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): assert set(used_disks) == {"jbod1"} assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" - node1.query("SYSTEM START MOVES {}").format(name) + node1.query("SYSTEM START MOVES {}".format(name)) time.sleep(3) used_disks = get_used_disks_for_table(node1, name) From 24b334258b31418f81ff8cbf8452a33289fb97bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 22:25:56 +0300 Subject: [PATCH 157/390] Resolve review comment --- src/Storages/StorageDistributed.cpp | 4 +++- src/Storages/StorageLog.cpp | 4 +++- src/Storages/StorageStripeLog.cpp | 4 +++- src/Storages/StorageTinyLog.cpp | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e573b264e4f..b06434b6317 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -57,6 +57,7 @@ #include #include #include +#include namespace @@ -866,7 +867,8 @@ void StorageDistributed::flushClusterNodesAllData() void StorageDistributed::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - if (!relative_data_path.empty() && relative_data_path != new_path_to_table_data) + assert(relative_data_path != new_path_to_table_data); + if (!relative_data_path.empty()) renameOnDisk(new_path_to_table_data); renameInMemory(new_table_id); } diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 8c76085e50c..e437bfb05f1 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -27,6 +27,8 @@ #include #include +#include + #define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin" #define DBMS_STORAGE_LOG_MARKS_FILE_NAME "__marks.mrk" @@ -548,7 +550,7 @@ void StorageLog::loadMarks() void StorageLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - if (table_path != new_path_to_table_data) + assert(table_path != new_path_to_table_data); { std::unique_lock lock(rwlock); diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c2bbb4e4bd8..c4344cf6f1f 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -35,6 +35,8 @@ #include #include +#include + namespace DB { @@ -282,7 +284,7 @@ StorageStripeLog::StorageStripeLog( void StorageStripeLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - if (table_path != new_path_to_table_data) + assert(table_path != new_path_to_table_data); { std::unique_lock lock(rwlock); diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 19b6d52b012..0bdcab8abf4 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -3,6 +3,7 @@ #include #include +#include #include @@ -407,7 +408,7 @@ void StorageTinyLog::addFiles(const String & column_name, const IDataType & type void StorageTinyLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id) { - if (table_path != new_path_to_table_data) + assert(table_path != new_path_to_table_data); { std::unique_lock lock(rwlock); From 7d0f3db63957a37b125d76ef04401f3dad17101d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 23:19:41 +0300 Subject: [PATCH 158/390] Fix annoying "Arcadia" --- src/Common/getMappedArea.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp index 921d95ab740..6e452f32b96 100644 --- a/src/Common/getMappedArea.cpp +++ b/src/Common/getMappedArea.cpp @@ -1,10 +1,10 @@ #include "getMappedArea.h" +#include #if defined(__linux__) #include #include -#include #include #include From 79674ea892048dd58b746662a2904af649c87d3f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 19 Sep 2020 00:11:09 +0300 Subject: [PATCH 159/390] Drop python from build dependencies I don't see where it can be required. And plus there is no python package in debian:testing anymore, only: - python2 - python3 So that said that you cannot build debian package on debian:testing anymore (but debian:stable don't have gcc9 for example). Everything would be great, if there will be packages for python2, i.e. python2-lxml, but there isn't, I guess because python2 had been deprecated long time ago... So it looks like converting to python3 is coming. Just for the history, now it is: - debian:stable -> buster - debian:testing -> bullseye --- debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/control b/debian/control index 58efd711d27..1014b8b0a3c 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,6 @@ Build-Depends: debhelper (>= 9), libicu-dev, libreadline-dev, gperf, - python, tzdata Standards-Version: 3.9.8 From dbc837c14858fd5d8921fee59b36d51f4b71d4e0 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 17 Sep 2020 12:58:34 +0300 Subject: [PATCH 160/390] IAccessStorage now can override the login() function. --- src/Access/AccessControlManager.cpp | 5 +++ src/Access/AccessControlManager.h | 1 + src/Access/ContextAccess.cpp | 17 -------- src/Access/ContextAccess.h | 3 -- src/Access/IAccessStorage.cpp | 62 ++++++++++++++++++++++++++++ src/Access/IAccessStorage.h | 17 ++++++++ src/Access/MultipleAccessStorage.cpp | 54 ++++++++++++++++++++++++ src/Access/MultipleAccessStorage.h | 2 + src/Interpreters/Context.cpp | 25 +++++------ src/Interpreters/Context.h | 4 +- 10 files changed, 155 insertions(+), 35 deletions(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 41137867213..dd711e93a95 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -339,6 +339,11 @@ void AccessControlManager::addStoragesFromMainConfig( } +UUID AccessControlManager::login(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const +{ + return MultipleAccessStorage::login(user_name, password, address, *external_authenticators); +} + void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) { external_authenticators->setConfig(config, getLogger()); diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index d7cf59cfb28..81a66ce8f1d 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -106,6 +106,7 @@ public: bool isSettingNameAllowed(const std::string_view & name) const; void checkSettingNameIsAllowed(const std::string_view & name) const; + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const; void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); std::shared_ptr getContextAccess( diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index d5e48baf110..fbc69e94a43 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -288,23 +288,6 @@ void ContextAccess::calculateAccessRights() const } -bool ContextAccess::isCorrectPassword(const String & password) const -{ - std::lock_guard lock{mutex}; - if (!user) - return false; - return user->authentication.isCorrectPassword(password, user_name, manager->getExternalAuthenticators()); -} - -bool ContextAccess::isClientHostAllowed() const -{ - std::lock_guard lock{mutex}; - if (!user) - return false; - return user->allowed_client_hosts.contains(params.address); -} - - UserPtr ContextAccess::getUser() const { std::lock_guard lock{mutex}; diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 9a5758b79a6..b42d50c3739 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -63,9 +63,6 @@ public: UserPtr getUser() const; String getUserName() const; - bool isCorrectPassword(const String & password) const; - bool isClientHostAllowed() const; - /// Returns information about current and enabled roles. /// The function can return nullptr. std::shared_ptr getRolesInfo() const; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 874ae612034..e5170221e18 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -13,6 +14,7 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_ALREADY_EXISTS; extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; + extern const int AUTHENTICATION_FAILED; extern const int LOGICAL_ERROR; } @@ -412,6 +414,57 @@ void IAccessStorage::notify(const Notifications & notifications) } +UUID IAccessStorage::login( + const String & user_name, + const String & password, + const Poco::Net::IPAddress & address, + const ExternalAuthenticators & external_authenticators) const +{ + return loginImpl(user_name, password, address, external_authenticators); +} + + +UUID IAccessStorage::loginImpl( + const String & user_name, + const String & password, + const Poco::Net::IPAddress & address, + const ExternalAuthenticators & external_authenticators) const +{ + if (auto id = find(user_name)) + { + if (auto user = tryRead(*id)) + { + if (isPasswordCorrectImpl(*user, password, external_authenticators) && isAddressAllowedImpl(*user, address)) + return *id; + } + } + throwCannotAuthenticate(user_name); +} + + +bool IAccessStorage::isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const +{ + return user.authentication.isCorrectPassword(password, user.getName(), external_authenticators); +} + + +bool IAccessStorage::isAddressAllowedImpl(const User & user, const Poco::Net::IPAddress & address) const +{ + return user.allowed_client_hosts.contains(address); +} + +UUID IAccessStorage::getIDOfLoggedUser(const String & user_name) const +{ + return getIDOfLoggedUserImpl(user_name); +} + + +UUID IAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const +{ + return getID(user_name); +} + + UUID IAccessStorage::generateRandomID() { static Poco::UUIDGenerator generator; @@ -500,4 +553,13 @@ void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & n "Cannot remove " + outputEntityTypeAndName(type, name) + " from " + getStorageName() + " because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } + + +void IAccessStorage::throwCannotAuthenticate(const String & user_name) +{ + /// We use the same message for all authentification failures because we don't want to give away any unnecessary information for security reasons, + /// only the log will show the exact reason. + throw Exception(user_name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); +} + } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 7851f8c9b6b..962ea9427ed 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -11,9 +11,13 @@ namespace Poco { class Logger; } +namespace Poco::Net { class IPAddress; } namespace DB { +struct User; +class ExternalAuthenticators; + /// Contains entities, i.e. instances of classes derived from IAccessEntity. /// The implementations of this class MUST be thread-safe. class IAccessStorage @@ -137,6 +141,14 @@ public: bool hasSubscription(EntityType type) const; bool hasSubscription(const UUID & id) const; + /// Finds an user, check its password and returns the ID of the user. + /// Throws an exception if no such user or password is incorrect. + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; + + /// Returns the ID of an user who has logged in (maybe on another node). + /// The function assumes that the password has been already checked somehow, so we can skip checking it now. + UUID getIDOfLoggedUser(const String & user_name) const; + protected: virtual std::optional findImpl(EntityType type, const String & name) const = 0; virtual std::vector findAllImpl(EntityType type) const = 0; @@ -151,6 +163,10 @@ protected: virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const = 0; virtual bool hasSubscriptionImpl(const UUID & id) const = 0; virtual bool hasSubscriptionImpl(EntityType type) const = 0; + virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; + virtual bool isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const; + virtual bool isAddressAllowedImpl(const User & user, const Poco::Net::IPAddress & address) const; + virtual UUID getIDOfLoggedUserImpl(const String & user_name) const; static UUID generateRandomID(); Poco::Logger * getLogger() const; @@ -165,6 +181,7 @@ protected: [[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; + [[noreturn]] static void throwCannotAuthenticate(const String & user_name); using Notification = std::tuple; using Notifications = std::vector; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index bf711b54d54..8ddc7410d8d 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -392,4 +392,58 @@ void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock added_subscriptions->clear(); } + +UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + { + try + { + auto id = storage->login(user_name, password, address, external_authenticators); + std::lock_guard lock{mutex}; + ids_cache.set(id, storage); + return id; + } + catch (...) + { + if (!storage->find(EntityType::USER, user_name)) + { + /// The authentication failed because there no users with such name in the `storage` + /// thus we can try to search in other nested storages. + continue; + } + throw; + } + } + throwCannotAuthenticate(user_name); +} + + +UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + { + try + { + auto id = storage->getIDOfLoggedUser(user_name); + std::lock_guard lock{mutex}; + ids_cache.set(id, storage); + return id; + } + catch (...) + { + if (!storage->find(EntityType::USER, user_name)) + { + /// The authentication failed because there no users with such name in the `storage` + /// thus we can try to search in other nested storages. + continue; + } + throw; + } + } + throwNotFound(EntityType::USER, user_name); +} + } diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 5d01894621f..36551f1cbc8 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -47,6 +47,8 @@ protected: ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; bool hasSubscriptionImpl(EntityType type) const override; + UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; + UUID getIDOfLoggedUserImpl(const String & user_name) const override; private: using Storages = std::vector; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 4210c3be1fc..d49ac48da80 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -106,7 +106,6 @@ namespace ErrorCodes extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; extern const int LOGICAL_ERROR; - extern const int AUTHENTICATION_FAILED; extern const int NOT_IMPLEMENTED; } @@ -689,22 +688,20 @@ void Context::setUserImpl(const String & name, const std::optional & pas client_info.current_password = password.value_or(""); #endif - auto new_user_id = getAccessControlManager().find(name); - std::shared_ptr new_access; - if (new_user_id) + /// Find a user with such name and check the password. + UUID new_user_id; + if (password) + new_user_id = getAccessControlManager().login(name, *password, address.host()); + else { - new_access = getAccessControlManager().getContextAccess(*new_user_id, {}, true, settings, current_database, client_info); /// Access w/o password is done under interserver-secret (remote_servers.secret) - /// So it is okay not to check client's host (since there is trust). - if (password && (!new_access->isClientHostAllowed() || !new_access->isCorrectPassword(*password))) - { - new_user_id = {}; - new_access = nullptr; - } + /// So it is okay not to check client's host in this case (since there is trust). + new_user_id = getAccessControlManager().getIDOfLoggedUser(name); } - if (!new_user_id || !new_access) - throw Exception(name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); + auto new_access = getAccessControlManager().getContextAccess( + new_user_id, /* current_roles = */ {}, /* use_default_roles = */ true, + settings, current_database, client_info); user_id = new_user_id; access = std::move(new_access); @@ -716,7 +713,7 @@ void Context::setUserImpl(const String & name, const std::optional & pas void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) { - setUserImpl(name, std::make_optional(password), address); + setUserImpl(name, password, address); } void Context::setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index b15ea018573..e0bd5606bd2 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -258,11 +258,13 @@ public: /// Sets the current user, checks the password and that the specified host is allowed. /// Must be called before getClientInfo. void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); - /// Sets the current user, *do not checks the password* but check that the specified host is allowed. + + /// Sets the current user, *do not checks the password and that the specified host is allowed*. /// Must be called before getClientInfo. /// /// (Used only internally in cluster, if the secret matches) void setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address); + void setQuotaKey(String quota_key_); UserPtr getUser() const; From 02acedd6a32149c39b709c6c298c8767b6f54006 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 17 Sep 2020 17:14:46 +0300 Subject: [PATCH 161/390] Use initial_address for quota calculation when the interserver-secret mode enabled. --- src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index d3a5ea38f3f..4d77759e517 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -965,7 +965,7 @@ void TCPHandler::receiveQuery() /// i.e. when the INSERT is done with the global context (w/o user). if (!client_info.initial_user.empty()) { - query_context->setUserWithoutCheckingPassword(client_info.initial_user, socket().peerAddress()); + query_context->setUserWithoutCheckingPassword(client_info.initial_user, client_info.initial_address); LOG_DEBUG(log, "User (initial): {}", query_context->getUserName()); } /// No need to update connection_context, since it does not requires user (it will not be used for query execution) From 4e77c2e48e7e4c5e56ae5dfccdc0216433804094 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 19 Sep 2020 01:29:31 +0300 Subject: [PATCH 162/390] Update MergeTreeDataMergerMutator.cpp --- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 48309303f59..91a34efc2b5 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1080,8 +1080,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor context_for_reading.setSetting("max_streams_to_max_threads_ratio", 1); context_for_reading.setSetting("max_threads", 1); /// Allow mutations to work when force_index_by_date or force_primary_key is on. - context_for_reading.setSetting("force_index_by_date", 0); - context_for_reading.setSetting("force_primary_key", 0); + context_for_reading.setSetting("force_index_by_date", Field(0)); + context_for_reading.setSetting("force_primary_key", Field(0)); MutationCommands commands_for_part; for (const auto & command : commands) From 37facf44825a35bc688c2de94f1db4644fab67da Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 17:44:46 +0300 Subject: [PATCH 163/390] Fix clang-tidy --- base/common/coverage.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/base/common/coverage.cpp b/base/common/coverage.cpp index 9f3c5ca653a..043f97f9593 100644 --- a/base/common/coverage.cpp +++ b/base/common/coverage.cpp @@ -3,12 +3,11 @@ #if WITH_COVERAGE # include - # include # if defined(__clang__) -extern "C" void __llvm_profile_dump(); +extern "C" void __llvm_profile_dump(); // NOLINT # elif defined(__GNUC__) || defined(__GNUG__) extern "C" void __gcov_exit(); # endif @@ -23,7 +22,7 @@ void dumpCoverageReportIfPossible() std::lock_guard lock(mutex); # if defined(__clang__) - __llvm_profile_dump(); + __llvm_profile_dump(); // NOLINT # elif defined(__GNUC__) || defined(__GNUG__) __gcov_exit(); # endif From ef5363b87f22ab35a52f5705d3b69c6dbc1621aa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 02:04:36 +0300 Subject: [PATCH 164/390] Fix terribly wrong code --- base/common/phdr_cache.cpp | 4 +- programs/server/Server.cpp | 1 - src/CMakeLists.txt | 1 + src/Common/SymbolIndex.cpp | 2 +- src/Common/tests/int_hashes_perf.cpp | 7 +- src/DataTypes/convertMySQLDataType.cpp | 126 ++++---- src/Disks/S3/registerDiskS3.cpp | 1 - src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/IO/S3/PocoHTTPClient.cpp | 19 +- src/IO/S3Common.cpp | 2 - ...est_aux_funcs_for_adaptive_granularity.cpp | 2 +- ...for_adaptive_granularity_compact_parts.cpp | 2 +- utils/db-generator/query_db_generator.cpp | 284 +++++++++--------- 13 files changed, 225 insertions(+), 228 deletions(-) diff --git a/base/common/phdr_cache.cpp b/base/common/phdr_cache.cpp index f362fb64285..4f6a066adab 100644 --- a/base/common/phdr_cache.cpp +++ b/base/common/phdr_cache.cpp @@ -14,7 +14,7 @@ # pragma clang diagnostic ignored "-Wunused-macros" #endif -#define __msan_unpoison(X, Y) +#define __msan_unpoison(X, Y) // NOLINT #if defined(__has_feature) # if __has_feature(memory_sanitizer) # undef __msan_unpoison @@ -84,7 +84,7 @@ extern "C" #ifdef ADDRESS_SANITIZER void __lsan_ignore_object(const void *); #else -void __lsan_ignore_object(const void *) {} +void __lsan_ignore_object(const void *) {} // NOLINT #endif } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index aa947b22593..97212e3ddcf 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6e8c395b26..8f84939c5a4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -67,6 +67,7 @@ set(dbms_sources) add_headers_and_sources(clickhouse_common_io Common) add_headers_and_sources(clickhouse_common_io Common/HashTable) add_headers_and_sources(clickhouse_common_io IO) +add_headers_and_sources(clickhouse_common_io IO/S3) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) if(USE_RDKAFKA) diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index ffa7f0462c9..a738512bb30 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -59,7 +59,7 @@ Otherwise you will get only exported symbols from program headers. # pragma clang diagnostic ignored "-Wunused-macros" #endif -#define __msan_unpoison_string(X) +#define __msan_unpoison_string(X) // NOLINT #if defined(__has_feature) # if __has_feature(memory_sanitizer) # undef __msan_unpoison_string diff --git a/src/Common/tests/int_hashes_perf.cpp b/src/Common/tests/int_hashes_perf.cpp index 7e8495cef27..569e9273bc4 100644 --- a/src/Common/tests/int_hashes_perf.cpp +++ b/src/Common/tests/int_hashes_perf.cpp @@ -187,11 +187,6 @@ static inline size_t tabulation(UInt64 x) return res; } -static inline size_t _intHash64(UInt64 x) -{ - return static_cast(intHash64(x)); -} - const size_t BUF_SIZE = 1024; @@ -284,7 +279,7 @@ int main(int argc, char ** argv) if (!method || method == 1) test (n, data.data(), "0: identity"); if (!method || method == 2) test (n, data.data(), "1: intHash32"); - if (!method || method == 3) test<_intHash64>(n, data.data(), "2: intHash64"); + if (!method || method == 3) test (n, data.data(), "2: intHash64"); if (!method || method == 4) test (n, data.data(), "3: two rounds"); if (!method || method == 5) test (n, data.data(), "4: two rounds and two variables"); if (!method || method == 6) test (n, data.data(), "5: two rounds with less ops"); diff --git a/src/DataTypes/convertMySQLDataType.cpp b/src/DataTypes/convertMySQLDataType.cpp index a509cf8b091..ef6fb7de874 100644 --- a/src/DataTypes/convertMySQLDataType.cpp +++ b/src/DataTypes/convertMySQLDataType.cpp @@ -40,74 +40,76 @@ DataTypePtr convertMySQLDataType(MultiEnum type_support, { // we expect mysql_data_type to be either "basic_type" or "type_with_params(param1, param2, ...)" auto data_type = std::string_view(mysql_data_type); - const auto param_start_pos = data_type.find("("); + const auto param_start_pos = data_type.find('('); const auto type_name = data_type.substr(0, param_start_pos); - DataTypePtr res = [&]() -> DataTypePtr { - if (type_name == "tinyint") - { - if (is_unsigned) - return std::make_shared(); - else - return std::make_shared(); - } - if (type_name == "smallint") - { - if (is_unsigned) - return std::make_shared(); - else - return std::make_shared(); - } - if (type_name == "int" || type_name == "mediumint") - { - if (is_unsigned) - return std::make_shared(); - else - return std::make_shared(); - } - if (type_name == "bigint") - { - if (is_unsigned) - return std::make_shared(); - else - return std::make_shared(); - } - if (type_name == "float") - return std::make_shared(); - if (type_name == "double") - return std::make_shared(); - if (type_name == "date") - return std::make_shared(); - if (type_name == "binary") - return std::make_shared(length); - if (type_name == "datetime" || type_name == "timestamp") - { - if (!type_support.isSet(MySQLDataTypesSupport::DATETIME64)) - return std::make_shared(); + DataTypePtr res; - if (type_name == "timestamp" && scale == 0) - { - return std::make_shared(); - } - else if (type_name == "datetime" || type_name == "timestamp") - { - return std::make_shared(scale); - } - } - - if (type_support.isSet(MySQLDataTypesSupport::DECIMAL) && (type_name == "numeric" || type_name == "decimal")) + if (type_name == "tinyint") + { + if (is_unsigned) + res = std::make_shared(); + else + res = std::make_shared(); + } + else if (type_name == "smallint") + { + if (is_unsigned) + res = std::make_shared(); + else + res = std::make_shared(); + } + else if (type_name == "int" || type_name == "mediumint") + { + if (is_unsigned) + res = std::make_shared(); + else + res = std::make_shared(); + } + else if (type_name == "bigint") + { + if (is_unsigned) + res = std::make_shared(); + else + res = std::make_shared(); + } + else if (type_name == "float") + res = std::make_shared(); + else if (type_name == "double") + res = std::make_shared(); + else if (type_name == "date") + res = std::make_shared(); + else if (type_name == "binary") + res = std::make_shared(length); + else if (type_name == "datetime" || type_name == "timestamp") + { + if (!type_support.isSet(MySQLDataTypesSupport::DATETIME64)) { - if (precision <= DecimalUtils::maxPrecision()) - return std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::maxPrecision()) - return std::make_shared>(precision, scale); - else if (precision <= DecimalUtils::maxPrecision()) - return std::make_shared>(precision, scale); + res = std::make_shared(); } - + else if (type_name == "timestamp" && scale == 0) + { + res = std::make_shared(); + } + else if (type_name == "datetime" || type_name == "timestamp") + { + res = std::make_shared(scale); + } + } + else if (type_support.isSet(MySQLDataTypesSupport::DECIMAL) && (type_name == "numeric" || type_name == "decimal")) + { + if (precision <= DecimalUtils::maxPrecision()) + res = std::make_shared>(precision, scale); + else if (precision <= DecimalUtils::maxPrecision()) + res = std::make_shared>(precision, scale); + else if (precision <= DecimalUtils::maxPrecision()) + res = std::make_shared>(precision, scale); + } + else + { /// Also String is fallback for all unknown types. - return std::make_shared(); - }(); + res = std::make_shared(); + } if (is_nullable) res = std::make_shared(res); diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index fbd19ce1cd9..1c7a5e24282 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -4,7 +4,6 @@ #include #include "DiskS3.h" #include "Disks/DiskCacheWrapper.h" -#include "Disks/DiskCacheWrapper.cpp" #include "Disks/DiskFactory.h" #include "ProxyConfiguration.h" #include "ProxyListConfiguration.h" diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 67a3cbabe6d..eb625374707 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -92,7 +92,7 @@ private: src_offset = src_offsets[i]; dst_offset += src_length; - if (src_length > 1 && dst_data[dst_offset - 2] != trailing_char_str.front()) + if (src_length > 1 && dst_data[dst_offset - 2] != UInt8(trailing_char_str.front())) { dst_data[dst_offset - 1] = trailing_char_str.front(); dst_data[dst_offset] = 0; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index e068f3581bd..6d601b1e9ed 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -101,7 +100,7 @@ void PocoHTTPClient::MakeRequestInternal( EnumSize, }; - auto selectMetric = [&request](S3MetricType type) + auto select_metric = [&request](S3MetricType type) { const ProfileEvents::Event events_map[][2] = { {ProfileEvents::S3ReadMicroseconds, ProfileEvents::S3WriteMicroseconds}, @@ -128,12 +127,12 @@ void PocoHTTPClient::MakeRequestInternal( throw Exception("Unsupported request method", ErrorCodes::NOT_IMPLEMENTED); }; - ProfileEvents::increment(selectMetric(S3MetricType::Count)); + ProfileEvents::increment(select_metric(S3MetricType::Count)); - const int MAX_REDIRECT_ATTEMPTS = 10; + static constexpr int max_redirect_attempts = 10; try { - for (int attempt = 0; attempt < MAX_REDIRECT_ATTEMPTS; ++attempt) + for (int attempt = 0; attempt < max_redirect_attempts; ++attempt) { Poco::URI poco_uri(uri); @@ -202,7 +201,7 @@ void PocoHTTPClient::MakeRequestInternal( auto & response_body_stream = session->receiveResponse(poco_response); watch.stop(); - ProfileEvents::increment(selectMetric(S3MetricType::Microseconds), watch.elapsedMicroseconds()); + ProfileEvents::increment(select_metric(S3MetricType::Microseconds), watch.elapsedMicroseconds()); int status_code = static_cast(poco_response.getStatus()); LOG_DEBUG(log, "Response status: {}, {}", status_code, poco_response.getReason()); @@ -214,7 +213,7 @@ void PocoHTTPClient::MakeRequestInternal( uri = location; LOG_DEBUG(log, "Redirecting request to new location: {}", location); - ProfileEvents::increment(selectMetric(S3MetricType::Redirects)); + ProfileEvents::increment(select_metric(S3MetricType::Redirects)); continue; } @@ -240,11 +239,11 @@ void PocoHTTPClient::MakeRequestInternal( if (status_code == 429 || status_code == 503) { // API throttling - ProfileEvents::increment(selectMetric(S3MetricType::Throttling)); + ProfileEvents::increment(select_metric(S3MetricType::Throttling)); } else { - ProfileEvents::increment(selectMetric(S3MetricType::Errors)); + ProfileEvents::increment(select_metric(S3MetricType::Errors)); } } else @@ -261,7 +260,7 @@ void PocoHTTPClient::MakeRequestInternal( response->SetClientErrorType(Aws::Client::CoreErrors::NETWORK_CONNECTION); response->SetClientErrorMessage(getCurrentExceptionMessage(false)); - ProfileEvents::increment(selectMetric(S3MetricType::Errors)); + ProfileEvents::increment(select_metric(S3MetricType::Errors)); } } } diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index ba9a61ab922..db7aaf1549b 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -12,9 +12,7 @@ # include # include # include -# include # include -# include # include # include # include diff --git a/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp b/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp index d9ddb8e9722..4e7bff0ef41 100644 --- a/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp +++ b/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp @@ -3,7 +3,7 @@ #include // I know that inclusion of .cpp is not good at all -#include +#include // NOLINT using namespace DB; static Block getBlockWithSize(size_t required_size_in_bytes, size_t size_of_row_in_bytes) diff --git a/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity_compact_parts.cpp b/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity_compact_parts.cpp index f87293dcd5d..09b24c7dad6 100644 --- a/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity_compact_parts.cpp +++ b/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity_compact_parts.cpp @@ -3,7 +3,7 @@ #include // I know that inclusion of .cpp is not good at all -#include +#include // NOLINT using namespace DB; diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index 88f46325c72..16aa8aa7769 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -16,6 +15,7 @@ #include #include + using ColumnType = uint32_t; using TableAndColumn = std::pair; pcg64 rng; @@ -92,7 +92,7 @@ TableAndColumn get_table_a_column(const std::string & c) } -enum type : ColumnType +enum Type : ColumnType { i = 1, // int @@ -114,9 +114,11 @@ enum type : ColumnType }; -std::map type_definition = { - {type::i, "Int64"}, {type::f, "Float64"}, {type::s, "String"}, {type::d, "Date"}, {type::dt, "DateTime"}, {type::b, "UInt8"} +std::map type_definition = +{ + {Type::i, "Int64"}, {Type::f, "Float64"}, {Type::s, "String"}, {Type::d, "Date"}, {Type::dt, "DateTime"}, {Type::b, "UInt8"} }; + ColumnType time_type(std::string value) { if (value.length() == 12) @@ -124,14 +126,14 @@ ColumnType time_type(std::string value) for (size_t i : {5, 8}) { if (value[i] != '-') - return type::s; + return Type::s; } for (size_t i : {1, 2, 3, 4, 6, 7, 9, 10}) { if (!isdigit(value[i])) - return type::s; + return Type::s; } - return type::d; + return Type::d; } if (value.length() == 21) @@ -139,18 +141,18 @@ ColumnType time_type(std::string value) for (size_t i : {5, 8}) { if (value[i] != '-') - return type::s; + return Type::s; } for (size_t i : {14, 17}) { if (value[i] != '-') - return type::s; + return Type::s; } if (value[11] != '-') - return type::s; - return type::dt; + return Type::s; + return Type::dt; } - return type::s; + return Type::s; } // Casting inner clickhouse parser type to our type ColumnType type_cast(int t) @@ -164,21 +166,21 @@ ColumnType type_cast(int t) case 19: case 20: case 21: - return type::i; + return Type::i; case 3: - return type::f; + return Type::f; case 16: - return type::s; + return Type::s; case 17: - return type::a | type::all; + return Type::a | Type::all; case 18: - return type::t | type::all; + return Type::t | Type::all; } - return type::all; + return Type::all; } @@ -197,28 +199,28 @@ public: , is_array(is_a) {} std::string value{}; - ColumnType type = type::all; + ColumnType type = Type::all; bool is_array = false; }; std::map func_to_return_type = { - {"divide", FuncRet(type::f, "")}, {"e", FuncRet(type::f, "e()")}, {"pi", FuncRet(type::f, "pi()")}, {"exp", FuncRet(type::f, "")}, - {"log", FuncRet(type::f,"")}, {"exp2", FuncRet(type::f, "")}, {"log2", FuncRet(type::f, "")}, {"exp10", FuncRet(type::f, "")}, - {"log10", FuncRet(type::f, "")}, {"sqrt", FuncRet(type::f, "")}, {"cbrt", FuncRet(type::f, "")}, {"erf", FuncRet(type::f, "")}, - {"erfc", FuncRet(type::f, "")}, {"lgamma", FuncRet(type::f, "")}, {"tgamma", FuncRet(type::f, "")}, {"sin", FuncRet(type::f, "")}, - {"cos", FuncRet(type::f, "")}, {"tan", FuncRet(type::f, "")}, {"asin", FuncRet(type::f, "")}, {"acos", FuncRet(type::f, "")}, - {"atan", FuncRet(type::f, "")}, {"pow", FuncRet(type::f, "")}, {"splitbystring", FuncRet(type::s | type::a,"")}, - {"splitbychar", FuncRet(type::s | type::a, "")}, {"alphatokens", FuncRet(type::s | type::a, "")}, {"toyear", FuncRet(type::i, "")}, - {"tomonth", FuncRet(type::i, "")}, {"todayofmonth", FuncRet(type::i, "")}, {"tohour", FuncRet(type::dt, "")}, {"tominute", FuncRet(type::dt, "")}, - {"toseconds", FuncRet(type::dt, "")}, {"tounixtimestamp", FuncRet(type::i, "")}, {"tostartofyear", FuncRet(type::dt | type::d, "")}, - {"tostartofquater",FuncRet(type::dt | type::d, "")}, {"tostartofmonth", FuncRet(type::dt | type::d, "")}, {"tomonday", FuncRet(type::dt | type::d, "")}, - {"tostartoffiveminutes", FuncRet(type::dt, "")}, {"tostartoftenminutes", FuncRet(type::dt, "")}, {"tostartoffifteenminutes", FuncRet(type::dt, "")}, - {"tostartofinterval", FuncRet(type::dt, "")}, {"totime", FuncRet(type::dt, "")}, {"torelativemonthnum", FuncRet(type::i, "")}, - {"torelativeweeknum", FuncRet(type::i, "")}, {"torelativedaynum", FuncRet(type::i, "")}, {"torelativehournum", FuncRet(type::i, "")}, - {"torelativeminutenum", FuncRet(type::i, "")}, {"torelativesecondsnum", FuncRet(type::i, "")}, {"datediff", FuncRet(type::d | type::dt, "")}, - {"formatdatetime", FuncRet(type::s, "")}, {"now", FuncRet(type::dt | type::d, "now()")}, {"today", FuncRet(type::d | type::dt, "today()")}, - {"yesterday", FuncRet(type::d | type::dt, "yesterday()")} + {"divide", FuncRet(Type::f, "")}, {"e", FuncRet(Type::f, "e()")}, {"pi", FuncRet(Type::f, "pi()")}, {"exp", FuncRet(Type::f, "")}, + {"log", FuncRet(Type::f,"")}, {"exp2", FuncRet(Type::f, "")}, {"log2", FuncRet(Type::f, "")}, {"exp10", FuncRet(Type::f, "")}, + {"log10", FuncRet(Type::f, "")}, {"sqrt", FuncRet(Type::f, "")}, {"cbrt", FuncRet(Type::f, "")}, {"erf", FuncRet(Type::f, "")}, + {"erfc", FuncRet(Type::f, "")}, {"lgamma", FuncRet(Type::f, "")}, {"tgamma", FuncRet(Type::f, "")}, {"sin", FuncRet(Type::f, "")}, + {"cos", FuncRet(Type::f, "")}, {"tan", FuncRet(Type::f, "")}, {"asin", FuncRet(Type::f, "")}, {"acos", FuncRet(Type::f, "")}, + {"atan", FuncRet(Type::f, "")}, {"pow", FuncRet(Type::f, "")}, {"splitbystring", FuncRet(Type::s | Type::a,"")}, + {"splitbychar", FuncRet(Type::s | Type::a, "")}, {"alphatokens", FuncRet(Type::s | Type::a, "")}, {"toyear", FuncRet(Type::i, "")}, + {"tomonth", FuncRet(Type::i, "")}, {"todayofmonth", FuncRet(Type::i, "")}, {"tohour", FuncRet(Type::dt, "")}, {"tominute", FuncRet(Type::dt, "")}, + {"toseconds", FuncRet(Type::dt, "")}, {"tounixtimestamp", FuncRet(Type::i, "")}, {"tostartofyear", FuncRet(Type::dt | Type::d, "")}, + {"tostartofquater",FuncRet(Type::dt | Type::d, "")}, {"tostartofmonth", FuncRet(Type::dt | Type::d, "")}, {"tomonday", FuncRet(Type::dt | Type::d, "")}, + {"tostartoffiveminutes", FuncRet(Type::dt, "")}, {"tostartoftenminutes", FuncRet(Type::dt, "")}, {"tostartoffifteenminutes", FuncRet(Type::dt, "")}, + {"tostartofinterval", FuncRet(Type::dt, "")}, {"totime", FuncRet(Type::dt, "")}, {"torelativemonthnum", FuncRet(Type::i, "")}, + {"torelativeweeknum", FuncRet(Type::i, "")}, {"torelativedaynum", FuncRet(Type::i, "")}, {"torelativehournum", FuncRet(Type::i, "")}, + {"torelativeminutenum", FuncRet(Type::i, "")}, {"torelativesecondsnum", FuncRet(Type::i, "")}, {"datediff", FuncRet(Type::d | Type::dt, "")}, + {"formatdatetime", FuncRet(Type::s, "")}, {"now", FuncRet(Type::dt | Type::d, "now()")}, {"today", FuncRet(Type::d | Type::dt, "today()")}, + {"yesterday", FuncRet(Type::d | Type::dt, "yesterday()")} }; std::set func_args_same_types = { @@ -226,23 +228,23 @@ std::set func_args_same_types = { }; std::map func_to_param_type = { - {"tostartofminute", type::dt}, {"plus", type::i | type::f | type::d | type::dt}, {"multiply", type::i | type::f}, - {"minus", type::i | type::f | type::d | type::dt}, {"negate", type::i | type::f}, {"divide", type::i | type::f}, - {"abs", type::i | type::f}, {"gcd", type::i | type::f}, {"lcm", type::i | type::f}, {"bitnot", type::i}, {"bitshiftleft", type::i}, - {"bitshiftright", type::i}, {"bittest", type::i}, {"exp", type::i | type::f}, {"log", type::i | type::f}, - {"exp2", type::i | type::f}, {"log2", type::i | type::f}, {"exp10", type::i | type::f}, {"log10", type::i | type::f}, - {"sqrt", type::i | type::f}, {"cbrt", type::i | type::f}, {"erf", type::i | type::f}, {"erfc", type::i | type::f}, - {"lgamma", type::i | type::f}, {"tgamma", type::i | type::f}, {"sin", type::i | type::f}, {"cos", type::i | type::f}, - {"tan", type::i | type::f}, {"asin", type::i | type::f}, {"acos", type::i | type::f}, {"atan", type::i | type::f}, - {"pow", type::i | type::f}, {"arrayjoin", type::all | type::a}, {"substring", type::s}, {"splitbystring", type::s}, {"splitbychar", type::s}, - {"alphatokens", type::s}, {"toyear", type::d | type::dt}, {"tomonth", type::d | type::dt}, {"todayofmonth", type::d | type::dt}, {"tohour", type::dt}, - {"tominute", type::dt}, {"tosecond", type::dt}, {"touixtimestamp", type::dt}, {"tostartofyear", type::d | type::dt}, - {"tostartofquarter", type::d | type::dt}, {"tostartofmonth", type::d | type::dt}, {"tomonday", type::d | type::dt}, - {"tostartoffiveminute", type::dt}, {"tostartoftenminutes", type::dt}, {"tostartoffifteenminutes", type::d | type::dt}, - {"tostartofinterval", type::d | type::dt}, {"totime", type::d | type::dt}, {"torelativehonthnum", type::d | type::dt}, - {"torelativeweeknum", type::d | type::dt}, {"torelativedaynum", type::d | type::dt}, {"torelativehournum", type::d | type::dt}, - {"torelativeminutenum", type::d | type::dt}, {"torelativesecondnum", type::d | type::dt}, {"datediff", type::d | type::dt}, - {"formatdatetime", type::dt} + {"tostartofminute", Type::dt}, {"plus", Type::i | Type::f | Type::d | Type::dt}, {"multiply", Type::i | Type::f}, + {"minus", Type::i | Type::f | Type::d | Type::dt}, {"negate", Type::i | Type::f}, {"divide", Type::i | Type::f}, + {"abs", Type::i | Type::f}, {"gcd", Type::i | Type::f}, {"lcm", Type::i | Type::f}, {"bitnot", Type::i}, {"bitshiftleft", Type::i}, + {"bitshiftright", Type::i}, {"bittest", Type::i}, {"exp", Type::i | Type::f}, {"log", Type::i | Type::f}, + {"exp2", Type::i | Type::f}, {"log2", Type::i | Type::f}, {"exp10", Type::i | Type::f}, {"log10", Type::i | Type::f}, + {"sqrt", Type::i | Type::f}, {"cbrt", Type::i | Type::f}, {"erf", Type::i | Type::f}, {"erfc", Type::i | Type::f}, + {"lgamma", Type::i | Type::f}, {"tgamma", Type::i | Type::f}, {"sin", Type::i | Type::f}, {"cos", Type::i | Type::f}, + {"tan", Type::i | Type::f}, {"asin", Type::i | Type::f}, {"acos", Type::i | Type::f}, {"atan", Type::i | Type::f}, + {"pow", Type::i | Type::f}, {"arrayjoin", Type::all | Type::a}, {"substring", Type::s}, {"splitbystring", Type::s}, {"splitbychar", Type::s}, + {"alphatokens", Type::s}, {"toyear", Type::d | Type::dt}, {"tomonth", Type::d | Type::dt}, {"todayofmonth", Type::d | Type::dt}, {"tohour", Type::dt}, + {"tominute", Type::dt}, {"tosecond", Type::dt}, {"touixtimestamp", Type::dt}, {"tostartofyear", Type::d | Type::dt}, + {"tostartofquarter", Type::d | Type::dt}, {"tostartofmonth", Type::d | Type::dt}, {"tomonday", Type::d | Type::dt}, + {"tostartoffiveminute", Type::dt}, {"tostartoftenminutes", Type::dt}, {"tostartoffifteenminutes", Type::d | Type::dt}, + {"tostartofinterval", Type::d | Type::dt}, {"totime", Type::d | Type::dt}, {"torelativehonthnum", Type::d | Type::dt}, + {"torelativeweeknum", Type::d | Type::dt}, {"torelativedaynum", Type::d | Type::dt}, {"torelativehournum", Type::d | Type::dt}, + {"torelativeminutenum", Type::d | Type::dt}, {"torelativesecondnum", Type::d | Type::dt}, {"datediff", Type::d | Type::dt}, + {"formatdatetime", Type::dt} }; @@ -252,7 +254,7 @@ public: TableAndColumn name; std::set equals; std::set values; - ColumnType type = type::all; + ColumnType type = Type::all; bool is_array = false; Column() = default; @@ -260,7 +262,7 @@ public: explicit Column(const std::string & column_name) { name = std::make_pair("", column_name); - type = type::all; + type = Type::all; } void merge(Column other) @@ -275,15 +277,15 @@ public: void printType() const { - if (type & type::i) + if (type & Type::i) std::cout << "I"; - if (type & type::f) + if (type & Type::f) std::cout << "F"; - if (type & type::s) + if (type & Type::s) std::cout << "S"; - if (type & type::d) + if (type & Type::d) std::cout << "D"; - if (type & type::dt) + if (type & Type::dt) std::cout << "DT"; if (is_array) std::cout << "ARR"; @@ -307,22 +309,22 @@ public: std::string generateOneValue() const { - if (type & type::i) + if (type & Type::i) return randomInteger(); - if (type & type::f) + if (type & Type::f) return randomFloat(); - if (type & type::d) + if (type & Type::d) return randomDate(); - if (type & type::dt) + if (type & Type::dt) return randomDatetime(); - if (type & type::s) + if (type & Type::s) return "'" + randomString(rng() % 40) + "'"; - if (type & type::b) + if (type & Type::b) return "0"; return ""; @@ -332,7 +334,7 @@ public: { if (values.size() > 2 && amount == 0) return false; - while (values.size() < 1 or amount > 0) + while (values.empty() or amount > 0) { amount -= 1; if (is_array) @@ -357,18 +359,18 @@ public: void unifyType() { - if (type & type::i) - type = type::i; - else if (type & type::f) - type = type::f; - else if (type & type::d) - type = type::d; - else if (type & type::dt) - type = type::dt; - else if (type & type::s) - type = type::s; - else if (type & type::b) - type = type::b; + if (type & Type::i) + type = Type::i; + else if (type & Type::f) + type = Type::f; + else if (type & Type::d) + type = Type::d; + else if (type & Type::dt) + type = Type::dt; + else if (type & Type::s) + type = Type::s; + else if (type & Type::b) + type = Type::b; else throw std::runtime_error("Error in determination column type " + name.first + '.' + name.second); } @@ -381,13 +383,15 @@ decartMul( std::set & mul) { std::set> result; - for (auto v : prev) - for (auto m : mul) + for (const auto & v : prev) + { + for (const auto & m : mul) { std::vector tmp = v; tmp.push_back(m); result.insert(tmp); } + } return result; } @@ -438,7 +442,7 @@ public: { name = other.name; columns.insert(other.columns.begin(), other.columns.end()); - for (auto desc : other.column_description) + for (const auto & desc : other.column_description) column_description[desc.first].merge(desc.second); } @@ -576,7 +580,7 @@ public: void merge(TableList other) { - for (auto table : other.tables) + for (const auto & table : other.tables) tables[table.first].merge(table.second); nested.insert(other.nested.begin(), other.nested.end()); if (main_table.empty()) @@ -617,14 +621,14 @@ FuncRet arrayJoinFunc(DB::ASTPtr ch, std::map & columns) for (const auto & indent : indents) { auto c = Column(indent); - c.type = type::all; + c.type = Type::all; c.is_array = true; if (columns.count(indent)) columns[indent].merge(c); else columns[indent] = c; } - FuncRet r(type::all, ""); + FuncRet r(Type::all, ""); return r; } return FuncRet(); @@ -637,7 +641,7 @@ FuncRet inFunc(DB::ASTPtr ch, std::map & columns) { std::set indents{}; std::set values{}; - ColumnType type_value = type::all; + ColumnType type_value = Type::all; for (auto & arg : x->arguments->children) { @@ -656,20 +660,20 @@ FuncRet inFunc(DB::ASTPtr ch, std::map & columns) for (auto & val : arr_values) { type = type_cast(val.getType()); - if (type == type::s || type == type::d || type == type::dt) + if (type == Type::s || type == Type::d || type == Type::dt) type = time_type(applyVisitor(DB::FieldVisitorToString(), val)); type_value &= type; values.insert(applyVisitor(DB::FieldVisitorToString(), val)); } }; - if (type & type::a) + if (type & Type::a) { auto arr_values = literal->value.get(); routine(arr_values); } - if (type & type::a) + if (type & Type::a) { auto arr_values = literal->value.get(); routine(arr_values); @@ -685,7 +689,7 @@ FuncRet inFunc(DB::ASTPtr ch, std::map & columns) else f = handlers[""]; FuncRet ret = f(arg, columns); - if (ret.value != "") + if (!ret.value.empty()) { values.insert(ret.value); } @@ -703,7 +707,7 @@ FuncRet inFunc(DB::ASTPtr ch, std::map & columns) else columns[indent] = c; } - FuncRet r(type::b | type::i, ""); + FuncRet r(Type::b | Type::i, ""); return r; } return FuncRet(); @@ -716,7 +720,7 @@ FuncRet arrayFunc(DB::ASTPtr ch, std::map & columns) { std::set indents = {}; std::string value = "["; - ColumnType type_value = type::i | type::f | type::d | type::dt | type::s; + ColumnType type_value = Type::i | Type::f | Type::d | Type::dt | Type::s; bool no_indent = true; for (const auto & arg : x->arguments->children) { @@ -730,7 +734,7 @@ FuncRet arrayFunc(DB::ASTPtr ch, std::map & columns) if (literal) { ColumnType type = type_cast(literal->value.getType()); - if (type == type::s || type == type::d || type == type::dt) + if (type == Type::s || type == Type::d || type == Type::dt) type = time_type(value); type_value &= type; @@ -764,7 +768,7 @@ FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) { std::set indents = {}; std::set values = {}; - ColumnType type_value = type::i | type::f | type::d | type::dt; + ColumnType type_value = Type::i | Type::f | Type::d | Type::dt; ColumnType args_types = 0; bool no_indent = true; for (auto & arg : x->arguments->children) @@ -793,11 +797,11 @@ FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) } args_types |= type; } - if (args_types & (type::d | type::dt)) - type_value -= type::f; - if (args_types & type::f) - type_value -= type::d | type::dt; - for (auto indent : indents) + if (args_types & (Type::d | Type::dt)) + type_value -= Type::f; + if (args_types & Type::f) + type_value -= Type::d | Type::dt; + for (const auto & indent : indents) { auto c = Column(indent); c.type = type_value; @@ -807,14 +811,14 @@ FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) columns[indent] = c; } ColumnType ret_type = 0; - if (args_types & type::dt) - ret_type = type::dt; - else if (args_types & type::d) - ret_type = type::d | type::dt; - else if (args_types & type::f) - ret_type = type::f; + if (args_types & Type::dt) + ret_type = Type::dt; + else if (args_types & Type::d) + ret_type = Type::d | Type::dt; + else if (args_types & Type::f) + ret_type = Type::f; else - ret_type = type::d | type::f | type::dt | type::i; + ret_type = Type::d | Type::f | Type::dt | Type::i; FuncRet r(ret_type, ""); if (no_indent) { @@ -833,7 +837,7 @@ FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) { std::set indents = {}; std::set values = {}; - ColumnType type_value = type::s; + ColumnType type_value = Type::s; for (auto & arg : x->arguments->children) { auto ident = std::dynamic_pointer_cast(arg); @@ -866,7 +870,7 @@ FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) else columns[indent] = c; } - FuncRet r(type::b, ""); + FuncRet r(Type::b, ""); return r; } return FuncRet(); @@ -874,22 +878,22 @@ FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) { - auto X = std::dynamic_pointer_cast(ch); - if (X) + auto x = std::dynamic_pointer_cast(ch); + if (x) { std::set indents = {}; std::set values = {}; - ColumnType type_value = type::all; + ColumnType type_value = Type::all; bool is_array = false; bool no_indent = true; - if (func_to_param_type.count(boost::algorithm::to_lower_copy(X->name))) + if (func_to_param_type.count(boost::algorithm::to_lower_copy(x->name))) { - type_value &= func_to_param_type[boost::algorithm::to_lower_copy(X->name)]; - is_array = func_to_param_type[boost::algorithm::to_lower_copy(X->name)] & type::a; + type_value &= func_to_param_type[boost::algorithm::to_lower_copy(x->name)]; + is_array = func_to_param_type[boost::algorithm::to_lower_copy(x->name)] & Type::a; } - for (auto arg : X->arguments->children) + for (const auto & arg : x->arguments->children) { - ColumnType type = type::all; + ColumnType type = Type::all; std::string value; auto ident = std::dynamic_pointer_cast(arg); if (ident) @@ -902,7 +906,7 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) { value = applyVisitor(DB::FieldVisitorToString(), literal->value); type = type_cast(literal->value.getType()); - is_array |= type & type::a; + is_array |= type & Type::a; } auto subfunc = std::dynamic_pointer_cast(arg); if (subfunc) @@ -922,31 +926,31 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) } if (!value.empty()) { - if (type == type::i) + if (type == Type::i) { values.insert(value); values.insert(value + " + " + randomInteger(1, 10)); values.insert(value + " - " + randomInteger(1, 10)); } - if (type == type::f) + if (type == Type::f) { values.insert(value); values.insert(value + " + " + randomFloat(1, 10)); values.insert(value + " - " + randomFloat(1, 10)); } - if (type & type::s || type & type::d || type & type::dt) + if (type & Type::s || type & Type::d || type & Type::dt) { - if (type == type::s) + if (type == Type::s) type = time_type(value); - if (type == type::s) + if (type == Type::s) values.insert(value); - if (type & type::d) + if (type & Type::d) { values.insert(value); values.insert("toDate(" + value + ") + " + randomInteger(1, 10)); values.insert("toDate(" + value + ") - " + randomInteger(1, 10)); } - else if (type & type::dt) + else if (type & Type::dt) { values.insert(value); values.insert( @@ -956,7 +960,7 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) } } } - if (func_args_same_types.count(boost::algorithm::to_lower_copy(X->name))) + if (func_args_same_types.count(boost::algorithm::to_lower_copy(x->name))) type_value &= type; } for (const auto & indent : indents) @@ -965,7 +969,7 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) c.type = type_value; c.is_array = is_array; if (func_args_same_types.count( - boost::algorithm::to_lower_copy(X->name))) + boost::algorithm::to_lower_copy(x->name))) c.values = values; for (const auto & ind : indents) if (ind != indent) @@ -976,31 +980,31 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) else columns[indent] = c; } - if (func_to_return_type.count(boost::algorithm::to_lower_copy(X->name))) + if (func_to_return_type.count(boost::algorithm::to_lower_copy(x->name))) { if (no_indent) { std::ostringstream ss; formatAST(*ch, ss); - auto r = func_to_return_type[boost::algorithm::to_lower_copy(X->name)]; + auto r = func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; r.value = ss.str(); return r; } - return func_to_return_type[boost::algorithm::to_lower_copy(X->name)]; + return func_to_return_type[boost::algorithm::to_lower_copy(x->name)]; } else if (func_to_param_type.count( - boost::algorithm::to_lower_copy(X->name))) + boost::algorithm::to_lower_copy(x->name))) { if (no_indent) { std::ostringstream ss; formatAST(*ch, ss); return FuncRet( - func_to_param_type[boost::algorithm::to_lower_copy(X->name)], + func_to_param_type[boost::algorithm::to_lower_copy(x->name)], ss.str()); } return FuncRet( - func_to_param_type[boost::algorithm::to_lower_copy(X->name)], + func_to_param_type[boost::algorithm::to_lower_copy(x->name)], ""); } } @@ -1070,7 +1074,7 @@ connectedEqualityFind( std::set & visited) { std::set result; - for (auto & column : now.equals) + for (const auto & column : now.equals) if (!visited.count(column)) { visited.insert(column); @@ -1117,18 +1121,18 @@ unificateColumns( result[column.second.name.first + "." + column.second.name.second], result, visited); - for (auto c : equal) + for (const auto & c : equal) result[c.first + "." + c.second].equals = equal; } for (auto & column : result) - for (auto e : column.second.equals) + for (const auto & e : column.second.equals) column.second.merge(result[e.first + "." + e.second]); for (auto & column : result) { column.second.unifyType(); if (column.second.generateValues()) - for (auto e : column.second.equals) + for (const auto & e : column.second.equals) result[e.first + "." + e.second].merge(column.second); } @@ -1174,18 +1178,18 @@ void parseSelectQuery(DB::ASTPtr ast, TableList & all_tables) for (auto & child : x->children) { auto ch = std::dynamic_pointer_cast(child); - auto TEast = std::dynamic_pointer_cast(ch->table_expression); - if (TEast && TEast->database_and_table_name) + auto table_expression_ast = std::dynamic_pointer_cast(ch->table_expression); + if (table_expression_ast && table_expression_ast->database_and_table_name) { - auto table_name = *(getIndent(TEast->database_and_table_name).begin()); + auto table_name = *(getIndent(table_expression_ast->database_and_table_name).begin()); all_tables.addTable(table_name); auto alias = getAlias(ch); if (!alias.empty()) all_tables.aliases[alias] = table_name; } - if (TEast && TEast->subquery) + if (table_expression_ast && table_expression_ast->subquery) { - for (auto select : getSelect(TEast->subquery)) + for (const auto & select : getSelect(table_expression_ast->subquery)) { TableList local; parseSelectQuery(select, local); From 1238822785124e7a37a9cfd73279ab471bd4ffbb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 13:37:40 +0300 Subject: [PATCH 165/390] Fix clang-tidy --- src/IO/S3/PocoHTTPClient.cpp | 2 +- src/IO/S3/PocoHTTPClient.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 6d601b1e9ed..0604c5f1360 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -78,7 +78,7 @@ std::shared_ptr PocoHTTPClient::MakeRequest( return response; } -void PocoHTTPClient::MakeRequestInternal( +void PocoHTTPClient::makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, Aws::Utils::RateLimits::RateLimiterInterface *, diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index a94a08e217d..eefc85fae70 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -37,7 +37,7 @@ public: Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const override; private: - void MakeRequestInternal( + void makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, From eb260f6a89d95646487f6421a8c2ca5829a86933 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 13:43:26 +0300 Subject: [PATCH 166/390] Fix bad code --- src/IO/S3/PocoHTTPClient.cpp | 7 +++++++ src/IO/S3/PocoHTTPClientFactory.cpp | 6 ++++++ src/IO/S3/PocoHTTPResponseStream.cpp | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 0604c5f1360..3735851931a 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -1,3 +1,7 @@ +#include + +#if USE_AWS_S3 + #include "PocoHTTPClient.h" #include @@ -14,6 +18,7 @@ #include #include + namespace ProfileEvents { extern const Event S3ReadMicroseconds; @@ -264,3 +269,5 @@ void PocoHTTPClient::makeRequestInternal( } } } + +#endif diff --git a/src/IO/S3/PocoHTTPClientFactory.cpp b/src/IO/S3/PocoHTTPClientFactory.cpp index 68f6a6b9823..b257f96e383 100644 --- a/src/IO/S3/PocoHTTPClientFactory.cpp +++ b/src/IO/S3/PocoHTTPClientFactory.cpp @@ -1,3 +1,7 @@ +#include + +#if USE_AWS_S3 + #include "PocoHTTPClientFactory.h" #include @@ -32,3 +36,5 @@ std::shared_ptr PocoHTTPClientFactory::CreateHttpRequest } } + +#endif diff --git a/src/IO/S3/PocoHTTPResponseStream.cpp b/src/IO/S3/PocoHTTPResponseStream.cpp index 0a198268f2e..b35188d9498 100644 --- a/src/IO/S3/PocoHTTPResponseStream.cpp +++ b/src/IO/S3/PocoHTTPResponseStream.cpp @@ -1,3 +1,8 @@ +#include + +#if USE_AWS_S3 + + #include "PocoHTTPResponseStream.h" #include @@ -10,3 +15,5 @@ PocoHTTPResponseStream::PocoHTTPResponseStream(std::shared_ptr Date: Thu, 17 Sep 2020 23:31:49 +0300 Subject: [PATCH 167/390] Avoid deadlocks in Log/TinyLog --- src/Storages/StorageTinyLog.cpp | 73 +++++++++++++++---- src/Storages/StorageTinyLog.h | 2 +- .../0_stateless/01499_log_deadlock.reference | 0 .../0_stateless/01499_log_deadlock.sql | 10 +++ 4 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 tests/queries/0_stateless/01499_log_deadlock.reference create mode 100644 tests/queries/0_stateless/01499_log_deadlock.sql diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 0bdcab8abf4..1ad8134cdbc 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -46,6 +46,7 @@ namespace DB namespace ErrorCodes { + extern const int TIMEOUT_EXCEEDED; extern const int DUPLICATE_COLUMN; extern const int INCORRECT_FILE_NAME; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -66,10 +67,19 @@ public: return Nested::flatten(res); } - TinyLogSource(size_t block_size_, const NamesAndTypesList & columns_, StorageTinyLog & storage_, size_t max_read_buffer_size_) + TinyLogSource( + size_t block_size_, + const NamesAndTypesList & columns_, + StorageTinyLog & storage_, + std::shared_lock && lock_, + size_t max_read_buffer_size_) : SourceWithProgress(getHeader(columns_)) - , block_size(block_size_), columns(columns_), storage(storage_), lock(storage_.rwlock) - , max_read_buffer_size(max_read_buffer_size_) {} + , block_size(block_size_), columns(columns_), storage(storage_), lock(std::move(lock_)) + , max_read_buffer_size(max_read_buffer_size_) + { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + } String getName() const override { return "TinyLog"; } @@ -80,7 +90,7 @@ private: size_t block_size; NamesAndTypesList columns; StorageTinyLog & storage; - std::shared_lock lock; + std::shared_lock lock; bool is_finished = false; size_t max_read_buffer_size; @@ -110,9 +120,14 @@ private: class TinyLogBlockOutputStream final : public IBlockOutputStream { public: - explicit TinyLogBlockOutputStream(StorageTinyLog & storage_, const StorageMetadataPtr & metadata_snapshot_) - : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(storage_.rwlock) + explicit TinyLogBlockOutputStream( + StorageTinyLog & storage_, + const StorageMetadataPtr & metadata_snapshot_, + std::unique_lock && lock_) + : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(std::move(lock_)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~TinyLogBlockOutputStream() override @@ -140,7 +155,7 @@ public: private: StorageTinyLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -410,7 +425,9 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); + std::unique_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); disk->moveDirectory(table_path, new_path_to_table_data); @@ -424,6 +441,16 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage } +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageTinyLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -437,27 +464,38 @@ Pipe StorageTinyLog::read( // When reading, we lock the entire storage, because we only have one file // per column and can't modify it concurrently. + const Settings & settings = context.getSettingsRef(); + return Pipe(std::make_shared( - max_block_size, Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, + Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), + *this, + std::shared_lock{rwlock, getLockTimeout(context)}, + settings.max_read_buffer_size)); } -BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - return std::make_shared(*this, metadata_snapshot); + return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}); } -CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } void StorageTinyLog::truncate( - const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) + const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context & context, TableExclusiveLockHolder &) { - std::unique_lock lock(rwlock); + std::unique_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); disk->clearDirectory(table_path); @@ -470,7 +508,10 @@ void StorageTinyLog::truncate( void StorageTinyLog::drop() { - std::unique_lock lock(rwlock); + std::unique_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + if (disk->exists(table_path)) disk->removeRecursive(table_path); files.clear(); diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index dc6ff101503..9c412e4dd14 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -70,7 +70,7 @@ private: Files files; FileChecker file_checker; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; diff --git a/tests/queries/0_stateless/01499_log_deadlock.reference b/tests/queries/0_stateless/01499_log_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01499_log_deadlock.sql b/tests/queries/0_stateless/01499_log_deadlock.sql new file mode 100644 index 00000000000..4765e22b023 --- /dev/null +++ b/tests/queries/0_stateless/01499_log_deadlock.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (x UInt8) ENGINE = TinyLog; + +SET max_execution_time = 1; +INSERT INTO t SELECT * FROM t; -- { serverError 159 } + +SET max_execution_time = 0, lock_acquire_timeout = 1; +INSERT INTO t SELECT * FROM t; -- { serverError 159 } + +DROP TABLE t; From dc7e3b75d3ad3c9c438ac0321a4cbc4c7c2d472a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 23:40:27 +0300 Subject: [PATCH 168/390] Whitespace --- src/Storages/StorageTinyLog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 1ad8134cdbc..84a17e88ed7 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -252,7 +252,7 @@ void TinyLogSource::readData(const String & name, const IDataType & type, IColum }; if (deserialize_states.count(name) == 0) - type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); + type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); type.deserializeBinaryBulkWithMultipleStreams(column, limit, settings, deserialize_states[name]); } From 3bb0088c86ff56747966fe5048ce6c5d8a0dc538 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 23:50:25 +0300 Subject: [PATCH 169/390] Whitespace --- src/Storages/StorageTinyLog.cpp | 16 ++++------------ src/Storages/StorageTinyLog.h | 2 +- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 84a17e88ed7..60ff7de2ea0 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -76,10 +76,10 @@ public: : SourceWithProgress(getHeader(columns_)) , block_size(block_size_), columns(columns_), storage(storage_), lock(std::move(lock_)) , max_read_buffer_size(max_read_buffer_size_) - { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - } + { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + } String getName() const override { return "TinyLog"; } @@ -493,10 +493,6 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context void StorageTinyLog::truncate( const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context & context, TableExclusiveLockHolder &) { - std::unique_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - disk->clearDirectory(table_path); files.clear(); @@ -508,10 +504,6 @@ void StorageTinyLog::truncate( void StorageTinyLog::drop() { - std::unique_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - if (disk->exists(table_path)) disk->removeRecursive(table_path); files.clear(); diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 9c412e4dd14..41af1e3ff0c 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -70,7 +70,7 @@ private: Files files; FileChecker file_checker; - mutable std::shared_timed_mutex rwlock; + mutable std::mutex mutex; Poco::Logger * log; From 8193da847b2c68e4bb2b0f4674e53da78e556cf4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 23:51:16 +0300 Subject: [PATCH 170/390] Simplification --- src/Storages/StorageTinyLog.cpp | 2 +- src/Storages/StorageTinyLog.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 60ff7de2ea0..cde7d6193f9 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -491,7 +491,7 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context } void StorageTinyLog::truncate( - const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context & context, TableExclusiveLockHolder &) + const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { disk->clearDirectory(table_path); diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 41af1e3ff0c..9c412e4dd14 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -70,7 +70,7 @@ private: Files files; FileChecker file_checker; - mutable std::mutex mutex; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; From 66ea3c0c43821dd3e57faaf894bca24e146d2ae0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 23:57:46 +0300 Subject: [PATCH 171/390] Remove useless method --- src/Storages/StorageTinyLog.cpp | 7 ------- src/Storages/StorageTinyLog.h | 2 -- 2 files changed, 9 deletions(-) diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index cde7d6193f9..ab1111c32ca 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -502,13 +502,6 @@ void StorageTinyLog::truncate( addFiles(column.name, *column.type); } -void StorageTinyLog::drop() -{ - if (disk->exists(table_path)) - disk->removeRecursive(table_path); - files.clear(); -} - void registerStorageTinyLog(StorageFactory & factory) { diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 9c412e4dd14..95b7d9f2941 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -43,8 +43,6 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) override; - void drop() override; - protected: StorageTinyLog( DiskPtr disk_, From bc8fc3e280c0e6afbd3c422c48b0fa775b79036b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 23:59:58 +0300 Subject: [PATCH 172/390] Simplification --- src/Storages/StorageLog.cpp | 4 ---- src/Storages/StorageStripeLog.cpp | 5 ----- 2 files changed, 9 deletions(-) diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index e437bfb05f1..c90611862e3 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -552,8 +552,6 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -569,8 +567,6 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { - std::shared_lock lock(rwlock); - files.clear(); file_count = 0; loaded_marks = false; diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c4344cf6f1f..21eebeab5e6 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -286,8 +286,6 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -359,10 +357,7 @@ CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Conte void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, const Context &, TableExclusiveLockHolder &) { - std::shared_lock lock(rwlock); - disk->clearDirectory(table_path); - file_checker = FileChecker{disk, table_path + "sizes.json"}; } From 111acdc63b9146093eb6b527531dc8598387c36c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 00:55:53 +0300 Subject: [PATCH 173/390] Continuation --- src/Storages/StorageFile.cpp | 40 +++++++++---- src/Storages/StorageFile.h | 2 +- src/Storages/StorageLog.cpp | 56 ++++++++++++++----- src/Storages/StorageLog.h | 4 +- src/Storages/StorageStripeLog.cpp | 40 ++++++++++--- src/Storages/StorageStripeLog.h | 2 +- src/Storages/StorageTinyLog.cpp | 4 -- .../0_stateless/01499_log_deadlock.reference | 2 + .../0_stateless/01499_log_deadlock.sql | 22 +++++++- 9 files changed, 129 insertions(+), 43 deletions(-) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index cc47047dc78..7b094f9bc06 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -52,6 +52,7 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int INCORRECT_FILE_NAME; extern const int FILE_DOESNT_EXIST; + extern const int TIMEOUT_EXCEEDED; } namespace @@ -199,6 +200,17 @@ StorageFile::StorageFile(CommonArguments args) setInMemoryMetadata(storage_metadata); } + +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + class StorageFileSource : public SourceWithProgress { public: @@ -245,7 +257,9 @@ public: { if (storage->use_table_fd) { - unique_lock = std::unique_lock(storage->rwlock); + unique_lock = std::unique_lock(storage->rwlock, getLockTimeout(context)); + if (!unique_lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); /// We could use common ReadBuffer and WriteBuffer in storage to leverage cache /// and add ability to seek unseekable files, but cache sync isn't supported. @@ -264,7 +278,9 @@ public: } else { - shared_lock = std::shared_lock(storage->rwlock); + shared_lock = std::shared_lock(storage->rwlock, getLockTimeout(context)); + if (!shared_lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } } @@ -373,8 +389,8 @@ private: bool finished_generate = false; - std::shared_lock shared_lock; - std::unique_lock unique_lock; + std::shared_lock shared_lock; + std::unique_lock unique_lock; }; @@ -417,7 +433,7 @@ Pipe StorageFile::read( for (size_t i = 0; i < num_streams; ++i) pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); return Pipe::unitePipes(std::move(pipes)); } @@ -429,12 +445,16 @@ public: explicit StorageFileBlockOutputStream( StorageFile & storage_, const StorageMetadataPtr & metadata_snapshot_, + std::unique_lock && lock_, const CompressionMethod compression_method, const Context & context) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + std::unique_ptr naked_buffer = nullptr; if (storage.use_table_fd) { @@ -488,7 +508,7 @@ public: private: StorageFile & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; std::unique_ptr write_buf; BlockOutputStreamPtr writer; bool prefix_written{false}; @@ -506,7 +526,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) path = paths[0]; - return std::make_shared(*this, metadata_snapshot, + return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}, chooseCompressionMethod(path, compression_method), context); } @@ -529,8 +549,6 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; - std::unique_lock lock(rwlock); - Poco::File(Poco::Path(path_new).parent()).createDirectories(); Poco::File(paths[0]).renameTo(path_new); @@ -547,8 +565,6 @@ void StorageFile::truncate( if (paths.size() != 1) throw Exception("Can't truncate table '" + getStorageID().getNameForLogs() + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); - std::unique_lock lock(rwlock); - if (use_table_fd) { if (0 != ::ftruncate(table_fd, 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index ea70dcd5311..babc56e3a11 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -89,7 +89,7 @@ private: std::atomic table_fd_was_used{false}; /// To detect repeating reads from stdin off_t table_fd_init_offset = -1; /// Initial position of fd, used for repeating reads - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log = &Poco::Logger::get("StorageFile"); }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index c90611862e3..d9b37f1ddee 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -39,6 +39,7 @@ namespace DB namespace ErrorCodes { + extern const int TIMEOUT_EXCEEDED; extern const int LOGICAL_ERROR; extern const int DUPLICATE_COLUMN; extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT; @@ -50,7 +51,6 @@ namespace ErrorCodes class LogSource final : public SourceWithProgress { public: - static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -116,13 +116,16 @@ private: class LogBlockOutputStream final : public IBlockOutputStream { public: - explicit LogBlockOutputStream(StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_) + explicit LogBlockOutputStream( + StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) , marks_stream( storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~LogBlockOutputStream() override @@ -149,7 +152,7 @@ public: private: StorageLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -507,9 +510,11 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) } -void StorageLog::loadMarks() +void StorageLog::loadMarks(std::chrono::seconds lock_timeout) { - std::unique_lock lock(rwlock); + std::unique_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); if (loaded_marks) return; @@ -606,6 +611,17 @@ const StorageLog::Marks & StorageLog::getMarksWithRealRowCount(const StorageMeta return it->second.marks; } + +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -616,11 +632,15 @@ Pipe StorageLog::read( unsigned num_streams) { metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); - loadMarks(); + + auto lock_timeout = getLockTimeout(context); + loadMarks(lock_timeout); NamesAndTypesList all_columns = Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)); - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); Pipes pipes; @@ -649,18 +669,28 @@ Pipe StorageLog::read( max_read_buffer_size)); } + /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe::unitePipes(std::move(pipes)); } -BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - loadMarks(); - return std::make_shared(*this, metadata_snapshot); + auto lock_timeout = getLockTimeout(context); + loadMarks(lock_timeout); + + std::unique_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + return std::make_shared(*this, metadata_snapshot, std::move(lock)); } -CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 49fc9a576c5..3553426b9e6 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -83,7 +83,7 @@ private: DiskPtr disk; String table_path; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Files files; @@ -104,7 +104,7 @@ private: /// Read marks files if they are not already read. /// It is done lazily, so that with a large number of tables, the server starts quickly. /// You can not call with a write locked `rwlock`. - void loadMarks(); + void loadMarks(std::chrono::seconds lock_timeout); /** For normal columns, the number of rows in the block is specified in the marks. * For array columns and nested structures, there are more than one group of marks that correspond to different files diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 21eebeab5e6..4567ac0d452 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -47,13 +47,13 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; + extern const int TIMEOUT_EXCEEDED; } class StripeLogSource final : public SourceWithProgress { public: - static Block getHeader( StorageStripeLog & storage, const StorageMetadataPtr & metadata_snapshot, @@ -157,10 +157,11 @@ private: class StripeLogBlockOutputStream final : public IBlockOutputStream { public: - explicit StripeLogBlockOutputStream(StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_) + explicit StripeLogBlockOutputStream( + StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) , data_out_file(storage.table_path + "data.bin") , data_out_compressed(storage.disk->writeFile(data_out_file, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append)) , data_out(std::make_unique( @@ -170,6 +171,8 @@ public: , index_out(std::make_unique(*index_out_compressed)) , block_out(*data_out, 0, metadata_snapshot->getSampleBlock(), false, index_out.get(), storage.disk->getFileSize(data_out_file)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~StripeLogBlockOutputStream() override @@ -223,7 +226,7 @@ public: private: StorageStripeLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; String data_out_file; std::unique_ptr data_out_compressed; @@ -295,6 +298,16 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora } +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageStripeLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -304,7 +317,9 @@ Pipe StorageStripeLog::read( const size_t /*max_block_size*/, unsigned num_streams) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); @@ -343,15 +358,22 @@ Pipe StorageStripeLog::read( } -BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - return std::make_shared(*this, metadata_snapshot); + std::unique_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + return std::make_shared(*this, metadata_snapshot, std::move(lock)); } -CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h index f88120a932e..ca3bfe4ff75 100644 --- a/src/Storages/StorageStripeLog.h +++ b/src/Storages/StorageStripeLog.h @@ -67,7 +67,7 @@ private: size_t max_compress_block_size; FileChecker file_checker; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; }; diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index ab1111c32ca..631284bd5cf 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -425,10 +425,6 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; diff --git a/tests/queries/0_stateless/01499_log_deadlock.reference b/tests/queries/0_stateless/01499_log_deadlock.reference index e69de29bb2d..91dea2c76e7 100644 --- a/tests/queries/0_stateless/01499_log_deadlock.reference +++ b/tests/queries/0_stateless/01499_log_deadlock.reference @@ -0,0 +1,2 @@ +6 +6 diff --git a/tests/queries/0_stateless/01499_log_deadlock.sql b/tests/queries/0_stateless/01499_log_deadlock.sql index 4765e22b023..700e89a4e4f 100644 --- a/tests/queries/0_stateless/01499_log_deadlock.sql +++ b/tests/queries/0_stateless/01499_log_deadlock.sql @@ -1,10 +1,30 @@ DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt8) ENGINE = TinyLog; -SET max_execution_time = 1; +SET max_execution_time = 1, lock_acquire_timeout = 1000; INSERT INTO t SELECT * FROM t; -- { serverError 159 } SET max_execution_time = 0, lock_acquire_timeout = 1; INSERT INTO t SELECT * FROM t; -- { serverError 159 } DROP TABLE t; + + +SET max_execution_time = 0, lock_acquire_timeout = 1000; + +CREATE TABLE t (x UInt8) ENGINE = Log; + +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; + +DROP TABLE t; + + +CREATE TABLE t (x UInt8) ENGINE = StripeLog; + +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; + +DROP TABLE t; From 4d06bc62b171c0a6e8291007b3ed7ef9af0ed4e2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 18 Sep 2020 01:09:27 +0300 Subject: [PATCH 174/390] Allow concurrent reads while writing to TinyLog --- src/Common/FileChecker.cpp | 5 +++ src/Common/FileChecker.h | 4 ++- src/Storages/StorageLog.cpp | 8 ++--- src/Storages/StorageStripeLog.cpp | 6 ++-- src/Storages/StorageTinyLog.cpp | 35 ++++++++++++------- .../0_stateless/01499_log_deadlock.reference | 1 + .../0_stateless/01499_log_deadlock.sql | 10 ++---- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index 6cbec3bda77..b306c3af990 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -41,6 +41,11 @@ void FileChecker::setEmpty(const String & full_file_path) map[fileName(full_file_path)] = 0; } +FileChecker::Map FileChecker::getFileSizes() const +{ + return map; +} + CheckResults FileChecker::check() const { // Read the files again every time you call `check` - so as not to violate the constancy. diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h index 015d4cadb07..59e7331952e 100644 --- a/src/Common/FileChecker.h +++ b/src/Common/FileChecker.h @@ -27,10 +27,12 @@ public: /// The purpose of this function is to rollback a group of unfinished writes. void repair(); -private: /// File name -> size. using Map = std::map; + Map getFileSizes() const; + +private: void initialize(); void updateImpl(const String & file_path); void load(Map & local_map, const String & path) const; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index d9b37f1ddee..2fbce21655c 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -512,7 +512,7 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) void StorageLog::loadMarks(std::chrono::seconds lock_timeout) { - std::unique_lock lock(rwlock, lock_timeout); + std::unique_lock lock(rwlock, lock_timeout); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); @@ -638,7 +638,7 @@ Pipe StorageLog::read( NamesAndTypesList all_columns = Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)); - std::shared_lock lock(rwlock, lock_timeout); + std::shared_lock lock(rwlock, lock_timeout); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); @@ -678,7 +678,7 @@ BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMe auto lock_timeout = getLockTimeout(context); loadMarks(lock_timeout); - std::unique_lock lock(rwlock, lock_timeout); + std::unique_lock lock(rwlock, lock_timeout); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); @@ -687,7 +687,7 @@ BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMe CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock, getLockTimeout(context)); + std::shared_lock lock(rwlock, getLockTimeout(context)); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 4567ac0d452..8ff8035c128 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -317,7 +317,7 @@ Pipe StorageStripeLog::read( const size_t /*max_block_size*/, unsigned num_streams) { - std::shared_lock lock(rwlock, getLockTimeout(context)); + std::shared_lock lock(rwlock, getLockTimeout(context)); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); @@ -360,7 +360,7 @@ Pipe StorageStripeLog::read( BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - std::unique_lock lock(rwlock, getLockTimeout(context)); + std::unique_lock lock(rwlock, getLockTimeout(context)); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); @@ -370,7 +370,7 @@ BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const Sto CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock, getLockTimeout(context)); + std::shared_lock lock(rwlock, getLockTimeout(context)); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 631284bd5cf..4d646c7451e 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -56,7 +57,6 @@ namespace ErrorCodes class TinyLogSource final : public SourceWithProgress { public: - static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -71,14 +71,12 @@ public: size_t block_size_, const NamesAndTypesList & columns_, StorageTinyLog & storage_, - std::shared_lock && lock_, - size_t max_read_buffer_size_) + size_t max_read_buffer_size_, + FileChecker::Map file_sizes_) : SourceWithProgress(getHeader(columns_)) - , block_size(block_size_), columns(columns_), storage(storage_), lock(std::move(lock_)) - , max_read_buffer_size(max_read_buffer_size_) + , block_size(block_size_), columns(columns_), storage(storage_) + , max_read_buffer_size(max_read_buffer_size_), file_sizes(std::move(file_sizes_)) { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } String getName() const override { return "TinyLog"; } @@ -90,19 +88,21 @@ private: size_t block_size; NamesAndTypesList columns; StorageTinyLog & storage; - std::shared_lock lock; bool is_finished = false; size_t max_read_buffer_size; + FileChecker::Map file_sizes; struct Stream { - Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_) + Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_, size_t file_size) : plain(disk->readFile(data_path, std::min(max_read_buffer_size_, disk->getFileSize(data_path)))), + limited(std::make_unique(*plain, file_size, false)), compressed(*plain) { } std::unique_ptr plain; + std::unique_ptr limited; CompressedReadBuffer compressed; }; @@ -246,7 +246,11 @@ void TinyLogSource::readData(const String & name, const IDataType & type, IColum String stream_name = IDataType::getFileNameForStream(name, path); if (!streams.count(stream_name)) - streams[stream_name] = std::make_unique(storage.disk, storage.files[stream_name].data_file_path, max_read_buffer_size); + { + String file_path = storage.files[stream_name].data_file_path; + streams[stream_name] = std::make_unique( + storage.disk, file_path, max_read_buffer_size, file_sizes[fileName(file_path)]); + } return &streams[stream_name]->compressed; }; @@ -462,12 +466,17 @@ Pipe StorageTinyLog::read( // per column and can't modify it concurrently. const Settings & settings = context.getSettingsRef(); + std::shared_lock lock{rwlock, getLockTimeout(context)}; + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe(std::make_shared( max_block_size, Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), *this, - std::shared_lock{rwlock, getLockTimeout(context)}, - settings.max_read_buffer_size)); + settings.max_read_buffer_size, + file_checker.getFileSizes())); } @@ -479,7 +488,7 @@ BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const Stora CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock, getLockTimeout(context)); + std::shared_lock lock(rwlock, getLockTimeout(context)); if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); diff --git a/tests/queries/0_stateless/01499_log_deadlock.reference b/tests/queries/0_stateless/01499_log_deadlock.reference index 91dea2c76e7..166be640db5 100644 --- a/tests/queries/0_stateless/01499_log_deadlock.reference +++ b/tests/queries/0_stateless/01499_log_deadlock.reference @@ -1,2 +1,3 @@ 6 6 +6 diff --git a/tests/queries/0_stateless/01499_log_deadlock.sql b/tests/queries/0_stateless/01499_log_deadlock.sql index 700e89a4e4f..e98b37f2455 100644 --- a/tests/queries/0_stateless/01499_log_deadlock.sql +++ b/tests/queries/0_stateless/01499_log_deadlock.sql @@ -1,17 +1,13 @@ DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt8) ENGINE = TinyLog; -SET max_execution_time = 1, lock_acquire_timeout = 1000; -INSERT INTO t SELECT * FROM t; -- { serverError 159 } - -SET max_execution_time = 0, lock_acquire_timeout = 1; -INSERT INTO t SELECT * FROM t; -- { serverError 159 } +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; DROP TABLE t; -SET max_execution_time = 0, lock_acquire_timeout = 1000; - CREATE TABLE t (x UInt8) ENGINE = Log; INSERT INTO t VALUES (1), (2), (3); From da87861285e63369bd79e176ce375a8d6ea18b85 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 15:15:47 +0300 Subject: [PATCH 175/390] Another test --- .../01502_log_tinylog_deadlock_race.reference | 6 ++ .../01502_log_tinylog_deadlock_race.sh | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference create mode 100755 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference new file mode 100644 index 00000000000..4bf85ae79f3 --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference @@ -0,0 +1,6 @@ +Testing TinyLog +Done TinyLog +Testing StripeLog +Done StripeLog +Testing Log +Done Log diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh new file mode 100755 index 00000000000..a5b2ff6db8f --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" + sleep 0.0$RANDOM + done +} + +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +export -f thread_create +export -f thread_drop +export -f thread_rename +export -f thread_select +export -f thread_insert +export -f thread_insert_select + + +# Do randomized queries and expect nothing extraordinary happens. + +function test_with_engine { + echo "Testing $1" + + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & + + wait + echo "Done $1" +} + +test_with_engine TinyLog +test_with_engine StripeLog +test_with_engine Log From 2264622f69f11c30e15fc31c678241cac2cc2b24 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 15:24:43 +0300 Subject: [PATCH 176/390] Fix clang-tidy --- src/IO/S3/PocoHTTPClient.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 3735851931a..c34d7719131 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -69,7 +69,7 @@ std::shared_ptr PocoHTTPClient::MakeRequest( Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const { auto response = Aws::MakeShared("PocoHTTPClient", request); - MakeRequestInternal(request, response, readLimiter, writeLimiter); + makeRequestInternal(request, response, readLimiter, writeLimiter); return response; } @@ -79,7 +79,7 @@ std::shared_ptr PocoHTTPClient::MakeRequest( Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const { auto response = Aws::MakeShared("PocoHTTPClient", request); - MakeRequestInternal(*request, response, readLimiter, writeLimiter); + makeRequestInternal(*request, response, readLimiter, writeLimiter); return response; } From 33efb36714e265d5ce5585de27bdf63fd7757294 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 3 Sep 2020 19:52:08 +0300 Subject: [PATCH 177/390] Use previous users.xml if failed to reload. --- src/Access/UsersConfigAccessStorage.cpp | 127 +++++++----------- src/Common/Config/ConfigReloader.cpp | 1 + tests/integration/helpers/test_tools.py | 16 +++ .../configs/config.d/text_log.xml | 3 - .../configs/{users.d => }/custom_settings.xml | 8 -- .../configs/illformed_setting.xml | 7 + .../integration/test_custom_settings/test.py | 32 ++--- .../__init__.py | 0 .../configs/changed_settings.xml | 9 ++ .../configs/normal_settings.xml | 9 ++ .../configs/unexpected_setting_enum.xml | 9 ++ .../configs/unexpected_setting_int.xml | 9 ++ .../configs/unknown_setting.xml | 8 ++ .../test.py | 90 +++++++++++++ 14 files changed, 217 insertions(+), 111 deletions(-) delete mode 100644 tests/integration/test_custom_settings/configs/config.d/text_log.xml rename tests/integration/test_custom_settings/configs/{users.d => }/custom_settings.xml (56%) create mode 100644 tests/integration/test_custom_settings/configs/illformed_setting.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/__init__.py create mode 100644 tests/integration/test_reloading_settings_from_users_xml/configs/changed_settings.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/configs/normal_settings.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_enum.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_int.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/configs/unknown_setting.xml create mode 100644 tests/integration/test_reloading_settings_from_users_xml/test.py diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 60bcc3784f3..ce10ebf0bcc 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -192,7 +192,7 @@ namespace } - std::vector parseUsers(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) + std::vector parseUsers(const Poco::Util::AbstractConfiguration & config) { Poco::Util::AbstractConfiguration::Keys user_names; config.keys("users", user_names); @@ -200,16 +200,8 @@ namespace std::vector users; users.reserve(user_names.size()); for (const auto & user_name : user_names) - { - try - { - users.push_back(parseUser(config, user_name)); - } - catch (...) - { - tryLogCurrentException(log, "Could not parse user " + backQuote(user_name)); - } - } + users.push_back(parseUser(config, user_name)); + return users; } @@ -256,12 +248,11 @@ namespace } quota->to_roles.add(user_ids); - return quota; } - std::vector parseQuotas(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) + std::vector parseQuotas(const Poco::Util::AbstractConfiguration & config) { Poco::Util::AbstractConfiguration::Keys user_names; config.keys("users", user_names); @@ -278,76 +269,63 @@ namespace quotas.reserve(quota_names.size()); for (const auto & quota_name : quota_names) { - try - { - auto it = quota_to_user_ids.find(quota_name); - const std::vector & quota_users = (it != quota_to_user_ids.end()) ? std::move(it->second) : std::vector{}; - quotas.push_back(parseQuota(config, quota_name, quota_users)); - } - catch (...) - { - tryLogCurrentException(log, "Could not parse quota " + backQuote(quota_name)); - } + auto it = quota_to_user_ids.find(quota_name); + const std::vector & quota_users = (it != quota_to_user_ids.end()) ? std::move(it->second) : std::vector{}; + quotas.push_back(parseQuota(config, quota_name, quota_users)); } return quotas; } - std::vector parseRowPolicies(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) + std::vector parseRowPolicies(const Poco::Util::AbstractConfiguration & config) { std::map, std::unordered_map> all_filters_map; + Poco::Util::AbstractConfiguration::Keys user_names; + config.keys("users", user_names); - try + for (const String & user_name : user_names) { - config.keys("users", user_names); - for (const String & user_name : user_names) + const String databases_config = "users." + user_name + ".databases"; + if (config.has(databases_config)) { - const String databases_config = "users." + user_name + ".databases"; - if (config.has(databases_config)) + Poco::Util::AbstractConfiguration::Keys database_keys; + config.keys(databases_config, database_keys); + + /// Read tables within databases + for (const String & database_key : database_keys) { - Poco::Util::AbstractConfiguration::Keys database_keys; - config.keys(databases_config, database_keys); + const String database_config = databases_config + "." + database_key; - /// Read tables within databases - for (const String & database_key : database_keys) + String database_name; + if (((database_key == "database") || (database_key.starts_with("database["))) && config.has(database_config + "[@name]")) + database_name = config.getString(database_config + "[@name]"); + else if (size_t bracket_pos = database_key.find('['); bracket_pos != std::string::npos) + database_name = database_key.substr(0, bracket_pos); + else + database_name = database_key; + + Poco::Util::AbstractConfiguration::Keys table_keys; + config.keys(database_config, table_keys); + + /// Read table properties + for (const String & table_key : table_keys) { - const String database_config = databases_config + "." + database_key; - - String database_name; - if (((database_key == "database") || (database_key.starts_with("database["))) && config.has(database_config + "[@name]")) - database_name = config.getString(database_config + "[@name]"); - else if (size_t bracket_pos = database_key.find('['); bracket_pos != std::string::npos) - database_name = database_key.substr(0, bracket_pos); + String table_config = database_config + "." + table_key; + String table_name; + if (((table_key == "table") || (table_key.starts_with("table["))) && config.has(table_config + "[@name]")) + table_name = config.getString(table_config + "[@name]"); + else if (size_t bracket_pos = table_key.find('['); bracket_pos != std::string::npos) + table_name = table_key.substr(0, bracket_pos); else - database_name = database_key; + table_name = table_key; - Poco::Util::AbstractConfiguration::Keys table_keys; - config.keys(database_config, table_keys); - - /// Read table properties - for (const String & table_key : table_keys) - { - String table_config = database_config + "." + table_key; - String table_name; - if (((table_key == "table") || (table_key.starts_with("table["))) && config.has(table_config + "[@name]")) - table_name = config.getString(table_config + "[@name]"); - else if (size_t bracket_pos = table_key.find('['); bracket_pos != std::string::npos) - table_name = table_key.substr(0, bracket_pos); - else - table_name = table_key; - - String filter_config = table_config + ".filter"; - all_filters_map[{database_name, table_name}][user_name] = config.getString(filter_config); - } + String filter_config = table_config + ".filter"; + all_filters_map[{database_name, table_name}][user_name] = config.getString(filter_config); } } } } - catch (...) - { - tryLogCurrentException(log, "Could not parse row policies"); - } std::vector policies; for (auto & [database_and_table_name, user_to_filters] : all_filters_map) @@ -450,23 +428,14 @@ namespace std::vector parseSettingsProfiles( const Poco::Util::AbstractConfiguration & config, - const std::function & check_setting_name_function, - Poco::Logger * log) + const std::function & check_setting_name_function) { std::vector profiles; Poco::Util::AbstractConfiguration::Keys profile_names; config.keys("profiles", profile_names); for (const auto & profile_name : profile_names) - { - try - { - profiles.push_back(parseSettingsProfile(config, profile_name, check_setting_name_function)); - } - catch (...) - { - tryLogCurrentException(log, "Could not parse profile " + backQuote(profile_name)); - } - } + profiles.push_back(parseSettingsProfile(config, profile_name, check_setting_name_function)); + return profiles; } } @@ -520,13 +489,13 @@ void UsersConfigAccessStorage::setConfig(const Poco::Util::AbstractConfiguration void UsersConfigAccessStorage::parseFromConfig(const Poco::Util::AbstractConfiguration & config) { std::vector> all_entities; - for (const auto & entity : parseUsers(config, getLogger())) + for (const auto & entity : parseUsers(config)) all_entities.emplace_back(generateID(*entity), entity); - for (const auto & entity : parseQuotas(config, getLogger())) + for (const auto & entity : parseQuotas(config)) all_entities.emplace_back(generateID(*entity), entity); - for (const auto & entity : parseRowPolicies(config, getLogger())) + for (const auto & entity : parseRowPolicies(config)) all_entities.emplace_back(generateID(*entity), entity); - for (const auto & entity : parseSettingsProfiles(config, check_setting_name_function, getLogger())) + for (const auto & entity : parseSettingsProfiles(config, check_setting_name_function)) all_entities.emplace_back(generateID(*entity), entity); memory_storage.setAll(all_entities); } diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index d4a2dfbafe5..677448e03ae 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -138,6 +138,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac if (throw_on_error) throw; tryLogCurrentException(log, "Error updating configuration from '" + path + "' config."); + return; } LOG_DEBUG(log, "Loaded config '{}', performed update on configuration", path); diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index d196142c518..9fbffe41819 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -60,3 +60,19 @@ def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_tim if expectation_tsv != val: raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join( expectation_tsv.diff(val, n1="expectation", n2="query")))) + +def assert_logs_contain(instance, substring): + if not instance.contains_in_log(substring): + raise AssertionError("'{}' not found in logs".format(substring)) + +def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_time=0.5): + for i in xrange(retry_count): + try: + if instance.contains_in_log(substring): + break + time.sleep(sleep_time) + except Exception as ex: + print "contains_in_log_with_retry retry {} exception {}".format(i + 1, ex) + time.sleep(sleep_time) + else: + raise AssertionError("'{}' not found in logs".format(substring)) diff --git a/tests/integration/test_custom_settings/configs/config.d/text_log.xml b/tests/integration/test_custom_settings/configs/config.d/text_log.xml deleted file mode 100644 index f386249f170..00000000000 --- a/tests/integration/test_custom_settings/configs/config.d/text_log.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/tests/integration/test_custom_settings/configs/users.d/custom_settings.xml b/tests/integration/test_custom_settings/configs/custom_settings.xml similarity index 56% rename from tests/integration/test_custom_settings/configs/users.d/custom_settings.xml rename to tests/integration/test_custom_settings/configs/custom_settings.xml index f32d0f3626d..d3865b434e6 100644 --- a/tests/integration/test_custom_settings/configs/users.d/custom_settings.xml +++ b/tests/integration/test_custom_settings/configs/custom_settings.xml @@ -6,13 +6,5 @@ Float64_-43.25e-1 'some text' - - - 1 - - - - 1 - diff --git a/tests/integration/test_custom_settings/configs/illformed_setting.xml b/tests/integration/test_custom_settings/configs/illformed_setting.xml new file mode 100644 index 00000000000..267978a8af9 --- /dev/null +++ b/tests/integration/test_custom_settings/configs/illformed_setting.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_custom_settings/test.py b/tests/integration/test_custom_settings/test.py index 32df79ec1e9..7e147f999a9 100644 --- a/tests/integration/test_custom_settings/test.py +++ b/tests/integration/test_custom_settings/test.py @@ -1,9 +1,10 @@ import pytest +import os from helpers.cluster import ClickHouseCluster +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"], - user_configs=["configs/users.d/custom_settings.xml"]) +node = cluster.add_instance('node') @pytest.fixture(scope="module", autouse=True) @@ -16,28 +17,17 @@ def started_cluster(): cluster.shutdown() -def test(): +def test_custom_settings(): + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/custom_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.query("SYSTEM RELOAD CONFIG") + assert node.query("SELECT getSetting('custom_a')") == "-5\n" assert node.query("SELECT getSetting('custom_b')") == "10000000000\n" assert node.query("SELECT getSetting('custom_c')") == "-4.325\n" assert node.query("SELECT getSetting('custom_d')") == "some text\n" - assert "custom_a = -5, custom_b = 10000000000, custom_c = -4.325, custom_d = \\'some text\\'" \ - in node.query("SHOW CREATE SETTINGS PROFILE default") - assert "no settings profile" in node.query_and_get_error( - "SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting") - assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_illformed_setting") - - -def test_invalid_settings(): - node.query("SYSTEM RELOAD CONFIG") - node.query("SYSTEM FLUSH LOGS") - - assert node.query("SELECT COUNT() FROM system.text_log WHERE" - " message LIKE '%Could not parse profile `profile_illformed_setting`%'" - " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n" - - assert node.query("SELECT COUNT() FROM system.text_log WHERE" - " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'" - " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n" +def test_illformed_setting(): + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/illformed_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + error_message = "Couldn't restore Field from dump: 1" + assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") diff --git a/tests/integration/test_reloading_settings_from_users_xml/__init__.py b/tests/integration/test_reloading_settings_from_users_xml/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_reloading_settings_from_users_xml/configs/changed_settings.xml b/tests/integration/test_reloading_settings_from_users_xml/configs/changed_settings.xml new file mode 100644 index 00000000000..382c2b2dc20 --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/configs/changed_settings.xml @@ -0,0 +1,9 @@ + + + + + 20000000000 + nearest_hostname + + + diff --git a/tests/integration/test_reloading_settings_from_users_xml/configs/normal_settings.xml b/tests/integration/test_reloading_settings_from_users_xml/configs/normal_settings.xml new file mode 100644 index 00000000000..85d1c26659f --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/configs/normal_settings.xml @@ -0,0 +1,9 @@ + + + + + 10000000000 + first_or_random + + + diff --git a/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_enum.xml b/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_enum.xml new file mode 100644 index 00000000000..ff2b40583de --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_enum.xml @@ -0,0 +1,9 @@ + + + + + 20000000000 + a + + + diff --git a/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_int.xml b/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_int.xml new file mode 100644 index 00000000000..4ef15ed3680 --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/configs/unexpected_setting_int.xml @@ -0,0 +1,9 @@ + + + + + a + nearest_hostname + + + diff --git a/tests/integration/test_reloading_settings_from_users_xml/configs/unknown_setting.xml b/tests/integration/test_reloading_settings_from_users_xml/configs/unknown_setting.xml new file mode 100644 index 00000000000..9bac09aef18 --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/configs/unknown_setting.xml @@ -0,0 +1,8 @@ + + + + + 8 + + + diff --git a/tests/integration/test_reloading_settings_from_users_xml/test.py b/tests/integration/test_reloading_settings_from_users_xml/test.py new file mode 100644 index 00000000000..b45568ee904 --- /dev/null +++ b/tests/integration/test_reloading_settings_from_users_xml/test.py @@ -0,0 +1,90 @@ +import pytest +import os +import time +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_retry + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', user_configs=["configs/normal_settings.xml"]) + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def reset_to_normal_settings_after_test(): + try: + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/normal_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.query("SYSTEM RELOAD CONFIG") + yield + finally: + pass + + +def test_force_reload(): + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" + + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.query("SYSTEM RELOAD CONFIG") + + assert node.query("SELECT getSetting('max_memory_usage')") == "20000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "nearest_hostname\n" + + +def test_reload_on_timeout(): + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" + + time.sleep(1) # The modification time of the 'z.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + + assert_eq_with_retry(node, "SELECT getSetting('max_memory_usage')", "20000000000") + assert_eq_with_retry(node, "SELECT getSetting('load_balancing')", "nearest_hostname") + + +def test_unknown_setting_force_reload(): + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + + error_message = "Setting xyz is neither a builtin setting nor started with the prefix 'custom_' registered for user-defined settings" + assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") + + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" + + +def test_unknown_setting_reload_on_timeout(): + time.sleep(1) # The modification time of the 'z.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + + error_message = "Setting xyz is neither a builtin setting nor started with the prefix 'custom_' registered for user-defined settings" + assert_logs_contain_with_retry(node, error_message) + + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" + + +def test_unexpected_setting_int(): + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), '/etc/clickhouse-server/users.d/z.xml') + error_message = "Cannot parse" + assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") + + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" + + +def test_unexpected_setting_enum(): + node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), '/etc/clickhouse-server/users.d/z.xml') + error_message = "Cannot parse" + assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") + + assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" + assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" From 46aa307d0ab958b3057064e1fbc6a1792a4c575a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 19 Sep 2020 17:19:06 +0300 Subject: [PATCH 178/390] fix EINVAL in renameat2 on MacOS --- src/Common/renameat2.cpp | 26 ++++++++++++++++++-------- src/Common/renameat2.h | 3 +++ src/Databases/DatabaseAtomic.cpp | 7 +++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 323b72267a6..5139a165c91 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -48,8 +48,10 @@ static bool supportsRenameat2Impl() #if defined(__NR_renameat2) -static void renameat2(const std::string & old_path, const std::string & new_path, int flags) +static bool renameat2(const std::string & old_path, const std::string & new_path, int flags) { + if(!supportsRenameat2()) + return false; if (old_path.empty() || new_path.empty()) throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR); @@ -57,7 +59,14 @@ static void renameat2(const std::string & old_path, const std::string & new_path /// int newdirfd (ignored for absolute newpath), const char *newpath, /// unsigned int flags if (0 == syscall(__NR_renameat2, AT_FDCWD, old_path.c_str(), AT_FDCWD, new_path.c_str(), flags)) - return; + return true; + + /// EINVAL means that filesystem does not support one of the flags. + /// It also may happen when running clickhouse in docker with Mac OS as a host OS. + /// supportsRenameat2() with uname is not enough in this case, because virtualized Linux kernel is used. + /// Other cases when EINVAL can be returned should never happen. + if (errno == EINVAL) + return false; if (errno == EEXIST) throwFromErrno("Cannot rename " + old_path + " to " + new_path + " because the second path already exists", ErrorCodes::ATOMIC_RENAME_FAIL); @@ -104,18 +113,19 @@ bool supportsRenameat2() void renameNoReplace(const std::string & old_path, const std::string & new_path) { - if (supportsRenameat2()) - renameat2(old_path, new_path, RENAME_NOREPLACE); - else + if (!renameat2(old_path, new_path, RENAME_NOREPLACE)) renameNoReplaceFallback(old_path, new_path); } void renameExchange(const std::string & old_path, const std::string & new_path) { - if (supportsRenameat2()) - renameat2(old_path, new_path, RENAME_EXCHANGE); - else + if (!renameat2(old_path, new_path, RENAME_EXCHANGE)) renameExchangeFallback(old_path, new_path); } +bool renameExchangeIfSupported(const std::string & old_path, const std::string & new_path) +{ + return renameat2(old_path, new_path, RENAME_EXCHANGE); +} + } diff --git a/src/Common/renameat2.h b/src/Common/renameat2.h index 333f85541f1..141c5d385c5 100644 --- a/src/Common/renameat2.h +++ b/src/Common/renameat2.h @@ -14,4 +14,7 @@ void renameNoReplace(const std::string & old_path, const std::string & new_path) /// Atomically exchange oldpath and newpath. Throw exception if some of them does not exist void renameExchange(const std::string & old_path, const std::string & new_path); +/// Returns false instead of throwing exception if renameat2 is not supported +bool renameExchangeIfSupported(const std::string & old_path, const std::string & new_path); + } diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index e0f72436422..ed17a8eccb1 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -277,7 +277,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path) { - bool check_file_exists = supportsRenameat2(); + bool check_file_exists = true; SCOPE_EXIT({ std::error_code code; if (check_file_exists) std::filesystem::remove(table_metadata_tmp_path, code); }); std::unique_lock lock{mutex}; @@ -286,9 +286,8 @@ void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & if (table_id.uuid != actual_table_id.uuid) throw Exception("Cannot alter table because it was renamed", ErrorCodes::CANNOT_ASSIGN_ALTER); - if (check_file_exists) - renameExchange(table_metadata_tmp_path, table_metadata_path); - else + check_file_exists = renameExchangeIfSupported(table_metadata_tmp_path, table_metadata_path); + if (!check_file_exists) std::filesystem::rename(table_metadata_tmp_path, table_metadata_path); } From 059646743bebc4c16ded71d65cb0e58c414b8f0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 17:20:00 +0300 Subject: [PATCH 179/390] Remove OpenCL --- CMakeLists.txt | 8 - cmake/find/opencl.cmake | 25 -- src/CMakeLists.txt | 5 - src/Columns/ColumnVector.cpp | 33 -- src/Columns/ColumnVector.h | 2 - src/Columns/IColumn.h | 11 - src/Common/BitonicSort.h | 221 ------------ src/Common/ErrorCodes.cpp | 1 - src/Common/oclBasics.h | 354 -------------------- src/Common/tests/CMakeLists.txt | 5 - src/Common/tests/bitonic_sort.cpp | 174 ---------- src/Core/Settings.h | 2 - src/Core/SettingsEnums.cpp | 5 - src/Core/SettingsEnums.h | 9 - src/Core/SortDescription.h | 10 +- src/Interpreters/InterpreterSelectQuery.cpp | 5 +- src/Interpreters/sortBlock.cpp | 7 +- 17 files changed, 7 insertions(+), 870 deletions(-) delete mode 100644 cmake/find/opencl.cmake delete mode 100644 src/Common/BitonicSort.h delete mode 100644 src/Common/oclBasics.h delete mode 100644 src/Common/tests/bitonic_sort.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cf97b2c40ff..a879ad9b076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -404,7 +404,6 @@ include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) include (cmake/find/termcap.cmake) # for external static llvm -include (cmake/find/opencl.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) @@ -450,13 +449,6 @@ include (cmake/find/mysqlclient.cmake) # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. -if (USE_OPENCL) - if (OS_DARWIN) - set(OPENCL_LINKER_FLAGS "-framework OpenCL") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OPENCL_LINKER_FLAGS}") - endif () -endif () - include (cmake/print_flags.cmake) if (TARGET global-group) diff --git a/cmake/find/opencl.cmake b/cmake/find/opencl.cmake deleted file mode 100644 index 2b0cc7c5dd4..00000000000 --- a/cmake/find/opencl.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: enable by default -if(0) - option(ENABLE_OPENCL "Enable OpenCL support" ${ENABLE_LIBRARIES}) -endif() - -if(NOT ENABLE_OPENCL) - return() -endif() - -# Intel OpenCl driver: sudo apt install intel-opencl-icd -# @sa https://github.com/intel/compute-runtime/releases - -# OpenCL applications should link with ICD loader -# sudo apt install opencl-headers ocl-icd-libopencl1 -# sudo ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so -# TODO: add https://github.com/OCL-dev/ocl-icd as submodule instead - -find_package(OpenCL) -if(OpenCL_FOUND) - set(USE_OPENCL 1) -else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable OpenCL support") -endif() - -message(STATUS "Using opencl=${USE_OPENCL}: ${OpenCL_INCLUDE_DIRS} : ${OpenCL_LIBRARIES}") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6e8c395b26..b058ab749b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -378,11 +378,6 @@ if (USE_BROTLI) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) endif() -if (USE_OPENCL) - target_link_libraries (clickhouse_common_io PRIVATE ${OpenCL_LIBRARIES}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${OpenCL_INCLUDE_DIRS}) -endif () - if (USE_CASSANDRA) dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY}) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 829dc7d7aaa..733a1510f93 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -19,15 +19,6 @@ #include -#if !defined(ARCADIA_BUILD) -# include -# if USE_OPENCL -# include "Common/BitonicSort.h" // Y_IGNORE -# endif -#else -#undef USE_OPENCL -#endif - #ifdef __SSE2__ #include #endif @@ -39,7 +30,6 @@ namespace ErrorCodes { extern const int PARAMETER_OUT_OF_BOUND; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; - extern const int OPENCL_ERROR; extern const int LOGICAL_ERROR; } @@ -147,29 +137,6 @@ namespace }; } -template -void ColumnVector::getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, - IColumn::SpecialSort special_sort) const -{ - if (special_sort == IColumn::SpecialSort::OPENCL_BITONIC) - { -#if !defined(ARCADIA_BUILD) -#if USE_OPENCL - if (!limit || limit >= data.size()) - { - res.resize(data.size()); - - if (data.empty() || BitonicSort::getInstance().sort(data, res, !reverse)) - return; - } -#else - throw DB::Exception("'special_sort = bitonic' specified but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR); -#endif -#endif - } - - getPermutation(reverse, limit, nan_direction_hint, res); -} template void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 55ab67d6214..c6600ca7e31 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -218,8 +218,6 @@ public: } void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override; - void getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, - IColumn::SpecialSort) const override; void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override; diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 40ff0649f4f..14e6a9d7eed 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -267,17 +267,6 @@ public: */ virtual void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const = 0; - enum class SpecialSort - { - NONE = 0, - OPENCL_BITONIC, - }; - - virtual void getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, SpecialSort) const - { - getPermutation(reverse, limit, nan_direction_hint, res); - } - /*in updatePermutation we pass the current permutation and the intervals at which it should be sorted * Then for each interval separately (except for the last one, if there is a limit) * We sort it based on data about the current column, and find all the intervals within this diff --git a/src/Common/BitonicSort.h b/src/Common/BitonicSort.h deleted file mode 100644 index 8140687c040..00000000000 --- a/src/Common/BitonicSort.h +++ /dev/null @@ -1,221 +0,0 @@ -#pragma once - -#include -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif - -#ifdef __APPLE__ -#include -#else -#include -#endif - -#include -#include -#include -#include -#include - -#include "oclBasics.h" -#include "bitonicSortKernels.cl" - -class BitonicSort -{ -public: - using KernelType = OCL::KernelType; - - enum Types - { - KernelInt8 = 0, - KernelUInt8, - KernelInt16, - KernelUInt16, - KernelInt32, - KernelUInt32, - KernelInt64, - KernelUInt64, - KernelMax - }; - - static BitonicSort & getInstance() - { - static BitonicSort instance = BitonicSort(); - return instance; - } - - /// Sorts given array in specified order. Returns `true` if given sequence was sorted, `false` otherwise. - template - bool sort(const DB::PaddedPODArray & data, DB::IColumn::Permutation & res, cl_uint sort_ascending [[maybe_unused]]) const - { - if constexpr ( - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v) - { - size_t data_size = data.size(); - - /// Getting the nearest power of 2. - size_t power = 8; - while (power < data_size) - power <<= 1; - - /// Allocates more space for additional stubs to be added if needed. - std::vector pairs_content(power); - std::vector pairs_indices(power); - - memcpy(&pairs_content[0], &data[0], sizeof(T) * data_size); - for (UInt32 i = 0; i < data_size; ++i) - pairs_indices[i] = i; - - fillWithStubs(pairs_content.data(), pairs_indices.data(), data_size, power - data_size, sort_ascending); - sort(pairs_content.data(), pairs_indices.data(), power, sort_ascending); - - for (size_t i = 0, shift = 0; i < power; ++i) - { - if (pairs_indices[i] >= data_size) - { - ++shift; - continue; - } - res[i - shift] = pairs_indices[i]; - } - - return true; - } - - return false; - } - - /// Creating a configuration instance with making all OpenCl required variables - /// such as device, platform, context, queue, program and kernel. - void configure() - { - OCL::Settings settings = OCL::Settings(1, nullptr, 1, nullptr, 1, 0); - - cl_platform_id platform = OCL::getPlatformID(settings); - cl_device_id device = OCL::getDeviceID(platform, settings); - cl_context gpu_context = OCL::makeContext(device, settings); - cl_command_queue command_queue = OCL::makeCommandQueue<2>(device, gpu_context, settings); - - cl_program program = OCL::makeProgram(bitonic_sort_kernels, gpu_context, device, settings); - - /// Creating kernels for each specified data type. - cl_int error = 0; - kernels.resize(KernelMax); - - kernels[KernelInt8] = std::shared_ptr(clCreateKernel(program, "bitonicSort_char", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelUInt8] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uchar", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelInt16] = std::shared_ptr(clCreateKernel(program, "bitonicSort_short", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelUInt16] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ushort", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelInt32] = std::shared_ptr(clCreateKernel(program, "bitonicSort_int", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelUInt32] = std::shared_ptr(clCreateKernel(program, "bitonicSort_uint", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelInt64] = std::shared_ptr(clCreateKernel(program, "bitonicSort_long", &error), clReleaseKernel); - OCL::checkError(error); - - kernels[KernelUInt64] = std::shared_ptr(clCreateKernel(program, "bitonicSort_ulong", &error), clReleaseKernel); - OCL::checkError(error); - - configuration = std::shared_ptr(new OCL::Configuration(device, gpu_context, command_queue, program)); - } - -private: - /// Dictionary with kernels for each type from list: uchar, char, ushort, short, uint, int, ulong and long. - std::vector> kernels; - /// Current configuration with core OpenCL instances. - std::shared_ptr configuration = nullptr; - - cl_kernel getKernel(Int8) const { return kernels[KernelInt8].get(); } - cl_kernel getKernel(UInt8) const { return kernels[KernelUInt8].get(); } - cl_kernel getKernel(Int16) const { return kernels[KernelInt16].get(); } - cl_kernel getKernel(UInt16) const { return kernels[KernelUInt16].get(); } - cl_kernel getKernel(Int32) const { return kernels[KernelInt32].get(); } - cl_kernel getKernel(UInt32) const { return kernels[KernelUInt32].get(); } - cl_kernel getKernel(Int64) const { return kernels[KernelInt64].get(); } - cl_kernel getKernel(UInt64) const { return kernels[KernelUInt64].get(); } - - /// Sorts p_input inplace with indices. Works only with arrays which size equals to power of two. - template - void sort(T * p_input, cl_uint * indices, cl_int array_size, cl_uint sort_ascending) const - { - cl_kernel kernel = getKernel(T(0)); - cl_int error = CL_SUCCESS; - cl_int num_stages = 0; - - for (cl_int temp = array_size; temp > 2; temp >>= 1) - num_stages++; - - /// Creating OpenCL buffers using input arrays memory. - cl_mem cl_input_buffer = OCL::createBuffer(p_input, array_size, configuration.get()->context()); - cl_mem cl_indices_buffer = OCL::createBuffer(indices, array_size, configuration.get()->context()); - - configureKernel(kernel, 0, static_cast(&cl_input_buffer)); - configureKernel(kernel, 1, static_cast(&cl_indices_buffer)); - configureKernel(kernel, 4, static_cast(&sort_ascending)); - - for (cl_int stage = 0; stage < num_stages; stage++) - { - configureKernel(kernel, 2, static_cast(&stage)); - - for (cl_int pass_of_stage = stage; pass_of_stage >= 0; pass_of_stage--) - { - configureKernel(kernel, 3, static_cast(&pass_of_stage)); - - /// Setting work-item dimensions. - size_t gsize = array_size / (2 * 4); - size_t global_work_size[1] = {pass_of_stage ? gsize : gsize << 1 }; // number of quad items in input array - - /// Executing kernel. - error = clEnqueueNDRangeKernel(configuration.get()->commandQueue(), kernel, 1, nullptr, - global_work_size, nullptr, 0, nullptr, nullptr); - OCL::checkError(error); - } - } - - /// Syncs all threads. - OCL::finishCommandQueue(configuration.get()->commandQueue()); - - OCL::releaseData(p_input, array_size, cl_input_buffer, configuration.get()->commandQueue()); - OCL::releaseData(indices, array_size, cl_indices_buffer, configuration.get()->commandQueue()); - } - - template - void configureKernel(cl_kernel kernel, int number_of_argument, void * source) const - { - cl_int error = clSetKernelArg(kernel, number_of_argument, sizeof(T), source); - OCL::checkError(error); - } - - /// Fills given sequences from `arraySize` index with `numberOfStubs` values. - template - void fillWithStubs(T * p_input, cl_uint * indices, cl_int array_size, cl_int number_of_stubs, cl_uint sort_ascending) const - { - T value = sort_ascending ? std::numeric_limits::max() : std::numeric_limits::min(); - for (cl_int index = 0; index < number_of_stubs; ++index) - { - p_input[array_size + index] = value; - indices[array_size + index] = array_size + index; - } - } - - BitonicSort() = default; - BitonicSort(BitonicSort const &) = delete; - void operator = (BitonicSort const &) = delete; -}; diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 85da23fb303..bf475bc9b21 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -486,7 +486,6 @@ namespace ErrorCodes extern const int NO_REMOTE_SHARD_AVAILABLE = 519; extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; - extern const int OPENCL_ERROR = 522; extern const int UNKNOWN_ROW_POLICY = 523; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524; extern const int INCORRECT_DISK_INDEX = 525; diff --git a/src/Common/oclBasics.h b/src/Common/oclBasics.h deleted file mode 100644 index a3e7636af1b..00000000000 --- a/src/Common/oclBasics.h +++ /dev/null @@ -1,354 +0,0 @@ -#pragma once - -#include -#if USE_OPENCL - -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif - -#ifdef __APPLE__ -#include -#else -#include -#endif - -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int OPENCL_ERROR; -} -} - -struct OCL -{ - using KernelType = std::remove_reference::type; - - /** - * Structure which represents the most essential settings of common OpenCl entities. - */ - struct Settings - { - // Platform info - cl_uint number_of_platform_entries; - cl_uint * number_of_available_platforms; - - // Devices info - cl_uint number_of_devices_entries; - cl_uint * number_of_available_devices; - - // Context settings - cl_context_properties * context_properties; - - void (* context_callback)(const char *, const void *, size_t, void *); - - void * context_callback_data; - - // Command queue settings - cl_command_queue_properties command_queue_properties; - - // Build settings - cl_uint number_of_program_source_pointers; - - void (* build_notification_routine)(cl_program, void *user_data); - - void * build_callback_data; - char * build_options; - - Settings(cl_uint number_of_platform_entries_, - cl_uint * number_of_available_platforms_, - cl_uint number_of_devices_entries_, - cl_uint * number_of_available_devices_, - cl_uint number_of_program_source_pointers_, - cl_command_queue_properties command_queue_properties_, - cl_context_properties * context_properties_ = nullptr, - void * context_data_callback_ = nullptr, - void (* context_callback_)(const char *, const void *, size_t, void *) = nullptr, - void (* build_notification_routine_)(cl_program, void * user_data) = nullptr, - void * build_callback_data_ = nullptr, - char * build_options_ = nullptr) - { - this->number_of_platform_entries = number_of_platform_entries_; - this->number_of_available_platforms = number_of_available_platforms_; - this->number_of_devices_entries = number_of_devices_entries_; - this->number_of_available_devices = number_of_available_devices_; - this->number_of_program_source_pointers = number_of_program_source_pointers_; - this->command_queue_properties = command_queue_properties_; - this->context_properties = context_properties_; - this->context_callback = context_callback_; - this->context_callback_data = context_data_callback_; - this->build_notification_routine = build_notification_routine_; - this->build_callback_data = build_callback_data_; - this->build_options = build_options_; - } - }; - - - /** - * Configuration with already created OpenCl common entities. - */ - class Configuration - { - public: - - Configuration(cl_device_id device, cl_context gpu_context, - cl_command_queue command_queue, cl_program program) - { - this->device_ = device; - this->gpu_context_ = std::shared_ptr(gpu_context, clReleaseContext); - this->command_queue_ = std::shared_ptr(command_queue, clReleaseCommandQueue); - this->program_ = std::shared_ptr(program, clReleaseProgram); - } - - cl_device_id device() { return device_; } - - cl_context context() { return gpu_context_.get(); } - - cl_command_queue commandQueue() { return command_queue_.get(); } - - cl_program program() { return program_.get(); } - - private: - - using ProgramType = std::remove_reference::type; - using CommandQueueType = std::remove_reference::type; - using ContextType = std::remove_reference::type; - - cl_device_id device_; - - std::shared_ptr gpu_context_; - std::shared_ptr command_queue_; - std::shared_ptr program_; - }; - - - static String opencl_error_to_str(cl_int error) - { -#define CASE_CL_CONSTANT(NAME) case NAME: return #NAME; - - // Suppose that no combinations are possible. - switch (error) - { - CASE_CL_CONSTANT(CL_SUCCESS) - CASE_CL_CONSTANT(CL_DEVICE_NOT_FOUND) - CASE_CL_CONSTANT(CL_DEVICE_NOT_AVAILABLE) - CASE_CL_CONSTANT(CL_COMPILER_NOT_AVAILABLE) - CASE_CL_CONSTANT(CL_MEM_OBJECT_ALLOCATION_FAILURE) - CASE_CL_CONSTANT(CL_OUT_OF_RESOURCES) - CASE_CL_CONSTANT(CL_OUT_OF_HOST_MEMORY) - CASE_CL_CONSTANT(CL_PROFILING_INFO_NOT_AVAILABLE) - CASE_CL_CONSTANT(CL_MEM_COPY_OVERLAP) - CASE_CL_CONSTANT(CL_IMAGE_FORMAT_MISMATCH) - CASE_CL_CONSTANT(CL_IMAGE_FORMAT_NOT_SUPPORTED) - CASE_CL_CONSTANT(CL_BUILD_PROGRAM_FAILURE) - CASE_CL_CONSTANT(CL_MAP_FAILURE) - CASE_CL_CONSTANT(CL_MISALIGNED_SUB_BUFFER_OFFSET) - CASE_CL_CONSTANT(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST) - CASE_CL_CONSTANT(CL_COMPILE_PROGRAM_FAILURE) - CASE_CL_CONSTANT(CL_LINKER_NOT_AVAILABLE) - CASE_CL_CONSTANT(CL_LINK_PROGRAM_FAILURE) - CASE_CL_CONSTANT(CL_DEVICE_PARTITION_FAILED) - CASE_CL_CONSTANT(CL_KERNEL_ARG_INFO_NOT_AVAILABLE) - CASE_CL_CONSTANT(CL_INVALID_VALUE) - CASE_CL_CONSTANT(CL_INVALID_DEVICE_TYPE) - CASE_CL_CONSTANT(CL_INVALID_PLATFORM) - CASE_CL_CONSTANT(CL_INVALID_DEVICE) - CASE_CL_CONSTANT(CL_INVALID_CONTEXT) - CASE_CL_CONSTANT(CL_INVALID_QUEUE_PROPERTIES) - CASE_CL_CONSTANT(CL_INVALID_COMMAND_QUEUE) - CASE_CL_CONSTANT(CL_INVALID_HOST_PTR) - CASE_CL_CONSTANT(CL_INVALID_MEM_OBJECT) - CASE_CL_CONSTANT(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR) - CASE_CL_CONSTANT(CL_INVALID_IMAGE_SIZE) - CASE_CL_CONSTANT(CL_INVALID_SAMPLER) - CASE_CL_CONSTANT(CL_INVALID_BINARY) - CASE_CL_CONSTANT(CL_INVALID_BUILD_OPTIONS) - CASE_CL_CONSTANT(CL_INVALID_PROGRAM) - CASE_CL_CONSTANT(CL_INVALID_PROGRAM_EXECUTABLE) - CASE_CL_CONSTANT(CL_INVALID_KERNEL_NAME) - CASE_CL_CONSTANT(CL_INVALID_KERNEL_DEFINITION) - CASE_CL_CONSTANT(CL_INVALID_KERNEL) - CASE_CL_CONSTANT(CL_INVALID_ARG_INDEX) - CASE_CL_CONSTANT(CL_INVALID_ARG_VALUE) - CASE_CL_CONSTANT(CL_INVALID_ARG_SIZE) - CASE_CL_CONSTANT(CL_INVALID_KERNEL_ARGS) - CASE_CL_CONSTANT(CL_INVALID_WORK_DIMENSION) - CASE_CL_CONSTANT(CL_INVALID_WORK_GROUP_SIZE) - CASE_CL_CONSTANT(CL_INVALID_WORK_ITEM_SIZE) - CASE_CL_CONSTANT(CL_INVALID_GLOBAL_OFFSET) - CASE_CL_CONSTANT(CL_INVALID_EVENT_WAIT_LIST) - CASE_CL_CONSTANT(CL_INVALID_EVENT) - CASE_CL_CONSTANT(CL_INVALID_OPERATION) - CASE_CL_CONSTANT(CL_INVALID_GL_OBJECT) - CASE_CL_CONSTANT(CL_INVALID_BUFFER_SIZE) - CASE_CL_CONSTANT(CL_INVALID_MIP_LEVEL) - CASE_CL_CONSTANT(CL_INVALID_GLOBAL_WORK_SIZE) - CASE_CL_CONSTANT(CL_INVALID_PROPERTY) - CASE_CL_CONSTANT(CL_INVALID_IMAGE_DESCRIPTOR) - CASE_CL_CONSTANT(CL_INVALID_COMPILER_OPTIONS) - CASE_CL_CONSTANT(CL_INVALID_LINKER_OPTIONS) - CASE_CL_CONSTANT(CL_INVALID_DEVICE_PARTITION_COUNT) - default: - return "UNKNOWN ERROR CODE "; - } - -#undef CASE_CL_CONSTANT - } - - - static void checkError(cl_int error) - { - if (error != CL_SUCCESS) - throw DB::Exception("OpenCL error: " + opencl_error_to_str(error), DB::ErrorCodes::OPENCL_ERROR); - } - - - /// Getting OpenCl main entities. - - static cl_platform_id getPlatformID(const Settings & settings) - { - cl_platform_id platform; - cl_int error = clGetPlatformIDs(settings.number_of_platform_entries, &platform, - settings.number_of_available_platforms); - checkError(error); - return platform; - } - - static cl_device_id getDeviceID(cl_platform_id & platform, const Settings & settings) - { - cl_device_id device; - cl_int error = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, settings.number_of_devices_entries, - &device, settings.number_of_available_devices); - OCL::checkError(error); - return device; - } - - static cl_context makeContext(cl_device_id & device, const Settings & settings) - { - cl_int error; - cl_context gpu_context = clCreateContext(settings.context_properties, settings.number_of_devices_entries, - &device, settings.context_callback, settings.context_callback_data, - &error); - OCL::checkError(error); - return gpu_context; - } - - template - static cl_command_queue makeCommandQueue(cl_device_id & device, cl_context & context, const Settings & settings [[maybe_unused]]) - { - cl_int error; - cl_command_queue command_queue; - - if constexpr (version == 1) - { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - command_queue = clCreateCommandQueue(context, device, settings.command_queue_properties, &error); -#pragma GCC diagnostic pop - } - else - { -#ifdef CL_VERSION_2_0 - command_queue = clCreateCommandQueueWithProperties(context, device, nullptr, &error); -#else - throw DB::Exception("Binary is built with OpenCL version < 2.0", DB::ErrorCodes::OPENCL_ERROR); -#endif - } - - OCL::checkError(error); - return command_queue; - } - - static cl_program makeProgram(const char * source_code, cl_context context, - cl_device_id device_id, const Settings & settings) - { - cl_int error = 0; - size_t source_size = strlen(source_code); - - cl_program program = clCreateProgramWithSource(context, settings.number_of_program_source_pointers, - &source_code, &source_size, &error); - checkError(error); - - error = clBuildProgram(program, settings.number_of_devices_entries, &device_id, settings.build_options, - settings.build_notification_routine, settings.build_callback_data); - - /// Combining additional logs output when program build failed. - if (error == CL_BUILD_PROGRAM_FAILURE) - { - size_t log_size; - error = clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, 0, nullptr, &log_size); - - checkError(error); - - std::vector log(log_size); - clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, log_size, log.data(), nullptr); - - checkError(error); - throw DB::Exception(log.data(), DB::ErrorCodes::OPENCL_ERROR); - } - - checkError(error); - return program; - } - - /// Configuring buffer for given input data - - template - static cl_mem createBuffer(K * p_input, cl_int array_size, cl_context context, cl_int elements_size = sizeof(K)) - { - cl_int error = CL_SUCCESS; - cl_mem cl_input_buffer = clCreateBuffer( - context, - CL_MEM_USE_HOST_PTR, - zeroCopySizeAlignment(elements_size * array_size), - p_input, - &error); - checkError(error); - return cl_input_buffer; - } - - static size_t zeroCopySizeAlignment(size_t required_size) - { - return required_size + (~required_size + 1) % 64; - } - - /// Manipulating with common OpenCL variables. - - static void finishCommandQueue(cl_command_queue command_queue) - { - // Blocks until all previously queued OpenCL commands in a queue are issued to the associated device. - cl_int error = clFinish(command_queue); - OCL::checkError(error); - } - - template - static void releaseData(T * origin, cl_int array_size, cl_mem cl_buffer, cl_command_queue command_queue, size_t offset = 0) - { - cl_int error = CL_SUCCESS; - - void * tmp_ptr = nullptr; - - // No events specified to be completed before enqueueing buffers, - // so `num_events_in_wait_list` passed with `0` value. - - tmp_ptr = clEnqueueMapBuffer(command_queue, cl_buffer, true, CL_MAP_READ, - offset, sizeof(cl_int) * array_size, 0, nullptr, nullptr, &error); - OCL::checkError(error); - if (tmp_ptr != origin) - throw DB::Exception("clEnqueueMapBuffer failed to return original pointer", DB::ErrorCodes::OPENCL_ERROR); - - error = clEnqueueUnmapMemObject(command_queue, cl_buffer, tmp_ptr, 0, nullptr, nullptr); - checkError(error); - - error = clReleaseMemObject(cl_buffer); - checkError(error); - } -}; - -#endif diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/tests/CMakeLists.txt index 8de9424e044..6a39c2f8553 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/tests/CMakeLists.txt @@ -35,11 +35,6 @@ add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io) target_include_directories(radix_sort SYSTEM PRIVATE ${PDQSORT_INCLUDE_DIR}) -if (USE_OPENCL) - add_executable (bitonic_sort bitonic_sort.cpp) - target_link_libraries (bitonic_sort PRIVATE clickhouse_common_io ${OPENCL_LINKER_FLAGS} ${OpenCL_LIBRARIES}) -endif () - add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) diff --git a/src/Common/tests/bitonic_sort.cpp b/src/Common/tests/bitonic_sort.cpp deleted file mode 100644 index 2545662c8cb..00000000000 --- a/src/Common/tests/bitonic_sort.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include -#include - -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif -#include -#include -#include -#include -#include -#include - -#include "Common/BitonicSort.h" - - -/// Generates vector of size 8 for testing. -/// Vector contains max possible value, min possible value and duplicate values. -template -static void generateTest(std::vector & data, Type min_value, Type max_value) -{ - int size = 10; - - data.resize(size); - data[0] = 10; - data[1] = max_value; - data[2] = 10; - data[3] = 20; - data[4] = min_value; - data[5] = min_value + 1; - data[6] = max_value - 5; - data[7] = 1; - data[8] = 42; - data[9] = max_value - 1; -} - - -static void check(const std::vector & indices, bool reverse = true) -{ - std::vector reference_indices{4, 5, 7, 0, 2, 3, 8, 6, 9, 1}; - if (reverse) std::reverse(reference_indices.begin(), reference_indices.end()); - - bool success = true; - for (size_t index = 0; index < reference_indices.size(); ++index) - { - if (indices[index] != reference_indices[index]) - { - success = false; - std::cerr << "Test failed. Reason: indices[" << index << "] = " - << indices[index] << ", it must be equal to " << reference_indices[index] << "\n"; - } - } - - std::string order_description = reverse ? "descending" : "ascending"; - std::cerr << "Sorted " << order_description << " sequence. Result: " << (success ? "Ok." : "Fail!") << "\n"; -} - - -template -static void sortBitonicSortWithPodArrays(const std::vector & data, std::vector & indices, bool ascending = true) -{ - DB::PaddedPODArray pod_array_data = DB::PaddedPODArray(data.size()); - DB::IColumn::Permutation pod_array_indices = DB::IColumn::Permutation(data.size()); - - for (size_t index = 0; index < data.size(); ++index) - { - *(pod_array_data.data() + index) = data[index]; - *(pod_array_indices.data() + index) = index; - } - - BitonicSort::getInstance().sort(pod_array_data, pod_array_indices, ascending); - - for (size_t index = 0; index < data.size(); ++index) - indices[index] = pod_array_indices[index]; -} - - -template -static void testBitonicSort(const std::string & test_name, Type min_value, Type max_value) -{ - std::cerr << test_name << std::endl; - - std::vector data; - generateTest(data, min_value, max_value); - - std::vector indices(data.size()); - - sortBitonicSortWithPodArrays(data, indices, true); - check(indices, false); - - sortBitonicSortWithPodArrays(data, indices, false); - check(indices, true); -} - - -static void straightforwardTests() -{ - testBitonicSort("Test 01: Int8.", CHAR_MIN, CHAR_MAX); - testBitonicSort("Test 02: UInt8.", 0, UCHAR_MAX); - testBitonicSort("Test 03: Int16.", SHRT_MIN, SHRT_MAX); - testBitonicSort("Test 04: UInt16.", 0, USHRT_MAX); - testBitonicSort("Test 05: Int32.", INT_MIN, INT_MAX); - testBitonicSort("Test 06: UInt32.", 0, UINT_MAX); - testBitonicSort("Test 07: Int64.", LONG_MIN, LONG_MAX); - testBitonicSort("Test 08: UInt64.", 0, ULONG_MAX); -} - - -template -static void bitonicSort(std::vector & data) -{ - size_t size = data.size(); - std::vector indices(size); - for (size_t i = 0; i < size; ++i) - indices[i] = i; - - sortBitonicSortWithPodArrays(data, indices); - - std::vector result(size); - for (size_t i = 0; i < size; ++i) - result[i] = data[indices[i]]; - - data = std::move(result); -} - - -template -static bool checkSort(const std::vector & data, size_t size) -{ - std::vector copy1(data.begin(), data.begin() + size); - std::vector copy2(data.begin(), data.begin() + size); - - std::sort(copy1.data(), copy1.data() + size); - bitonicSort(copy2); - - for (size_t i = 0; i < size; ++i) - if (copy1[i] != copy2[i]) - return false; - - return true; -} - - -int main() -{ - BitonicSort::getInstance().configure(); - - straightforwardTests(); - - size_t size = 1100; - std::vector data(size); - for (size_t i = 0; i < size; ++i) - data[i] = rand(); - - for (size_t i = 0; i < 128; ++i) - { - if (!checkSort(data, i)) - { - std::cerr << "fail at length " << i << std::endl; - return 1; - } - } - - for (size_t i = 128; i < size; i += 7) - { - if (!checkSort(data, i)) - { - std::cerr << "fail at length " << i << std::endl; - return 1; - } - } - - return 0; -} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5417bbad64c..b96b1b12c24 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -105,8 +105,6 @@ class IColumn; M(UInt64, parallel_replicas_count, 0, "", 0) \ M(UInt64, parallel_replica_offset, 0, "", 0) \ \ - M(SpecialSort, special_sort, SpecialSort::NOT_SPECIFIED, "Specifies a sorting algorithm which will be using in ORDER BY query.", 0) \ - \ M(Bool, skip_unavailable_shards, false, "If 1, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index c0d2906e2fc..b4db51a506d 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -23,11 +23,6 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING, {"round_robin", LoadBalancing::ROUND_ROBIN}}) -IMPLEMENT_SETTING_ENUM(SpecialSort, ErrorCodes::UNKNOWN_JOIN, - {{"not_specified", SpecialSort::NOT_SPECIFIED}, - {"opencl_bitonic", SpecialSort::OPENCL_BITONIC}}) - - IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN, {{"", JoinStrictness::Unspecified}, {"ALL", JoinStrictness::ALL}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 7ed5ffb0c35..426497fff78 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -47,15 +47,6 @@ enum class JoinAlgorithm DECLARE_SETTING_ENUM(JoinAlgorithm) -enum class SpecialSort -{ - NOT_SPECIFIED = 0, - OPENCL_BITONIC, -}; - -DECLARE_SETTING_ENUM(SpecialSort) - - /// Which rows should be included in TOTALS. enum class TotalsMode { diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 2198789b0b6..bd3b7bc45ff 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -32,22 +32,20 @@ struct SortColumnDescription std::shared_ptr collator; /// Collator for locale-specific comparison of strings bool with_fill; FillColumnDescription fill_description; - SpecialSort special_sort; - SortColumnDescription( size_t column_number_, int direction_, int nulls_direction_, - const std::shared_ptr & collator_ = nullptr, SpecialSort special_sort_ = SpecialSort::NOT_SPECIFIED, + const std::shared_ptr & collator_ = nullptr, bool with_fill_ = false, const FillColumnDescription & fill_description_ = {}) : column_number(column_number_), direction(direction_), nulls_direction(nulls_direction_), collator(collator_) - , with_fill(with_fill_), fill_description(fill_description_), special_sort(special_sort_) {} + , with_fill(with_fill_), fill_description(fill_description_) {} SortColumnDescription( const std::string & column_name_, int direction_, int nulls_direction_, - const std::shared_ptr & collator_ = nullptr, SpecialSort special_sort_ = SpecialSort::NOT_SPECIFIED, + const std::shared_ptr & collator_ = nullptr, bool with_fill_ = false, const FillColumnDescription & fill_description_ = {}) : column_name(column_name_), column_number(0), direction(direction_), nulls_direction(nulls_direction_) - , collator(collator_), with_fill(with_fill_), fill_description(fill_description_), special_sort(special_sort_) {} + , collator(collator_), with_fill(with_fill_), fill_description(fill_description_) {} bool operator == (const SortColumnDescription & other) const { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8f9f22e6ee2..22106387fc4 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -621,7 +621,6 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co { SortDescription order_descr; order_descr.reserve(query.orderBy()->children.size()); - SpecialSort special_sort = context.getSettings().special_sort.value; for (const auto & elem : query.orderBy()->children) { String name = elem->children.front()->getColumnName(); @@ -635,10 +634,10 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co { FillColumnDescription fill_desc = getWithFillDescription(order_by_elem, context); order_descr.emplace_back(name, order_by_elem.direction, - order_by_elem.nulls_direction, collator, special_sort, true, fill_desc); + order_by_elem.nulls_direction, collator, true, fill_desc); } else - order_descr.emplace_back(name, order_by_elem.direction, order_by_elem.nulls_direction, collator, special_sort); + order_descr.emplace_back(name, order_by_elem.direction, order_by_elem.nulls_direction, collator); } return order_descr; diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index d84708b9c57..c2436806fcd 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -132,12 +132,7 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) else if (!isColumnConst(*column)) { int nan_direction_hint = description[0].nulls_direction; - auto special_sort = description[0].special_sort; - - if (special_sort == SpecialSort::OPENCL_BITONIC) - column->getSpecialPermutation(reverse, limit, nan_direction_hint, perm, IColumn::SpecialSort::OPENCL_BITONIC); - else - column->getPermutation(reverse, limit, nan_direction_hint, perm); + column->getPermutation(reverse, limit, nan_direction_hint, perm); } else /// we don't need to do anything with const column From 772eccf03d70b197f360479477405f1120f0df4c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 17:47:22 +0300 Subject: [PATCH 180/390] Move code to .cpp --- src/Interpreters/QueryParameterVisitor.cpp | 46 ++++++++++++++++++++++ src/Interpreters/QueryParameterVisitor.h | 35 +++------------- src/Interpreters/ya.make | 1 + 3 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 src/Interpreters/QueryParameterVisitor.cpp diff --git a/src/Interpreters/QueryParameterVisitor.cpp b/src/Interpreters/QueryParameterVisitor.cpp new file mode 100644 index 00000000000..8a257830443 --- /dev/null +++ b/src/Interpreters/QueryParameterVisitor.cpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +QueryParameterVisitor::QueryParameterVisitor(NameSet & parameters_name) + : query_parameters(parameters_name) +{ +} + +void QueryParameterVisitor::visit(const ASTPtr & ast) +{ + for (const auto & child : ast->children) + { + if (const auto & query_parameter = child->as()) + visitQueryParameter(*query_parameter); + else + visit(child); + } +} + +void QueryParameterVisitor::visitQueryParameter(const ASTQueryParameter & query_parameter) +{ + query_parameters.insert(query_parameter.name); +} + +NameSet analyzeReceiveQueryParams(const std::string & query) +{ + NameSet query_params; + const char * query_begin = query.data(); + const char * query_end = query.data() + query.size(); + + ParserQuery parser(query_end, false); + ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "analyzeReceiveQueryParams", 0, 0); + QueryParameterVisitor(query_params).visit(extract_query_ast); + return query_params; +} + +} + diff --git a/src/Interpreters/QueryParameterVisitor.h b/src/Interpreters/QueryParameterVisitor.h index d3e618058c0..24ef5b08dcf 100644 --- a/src/Interpreters/QueryParameterVisitor.h +++ b/src/Interpreters/QueryParameterVisitor.h @@ -2,48 +2,25 @@ #include #include -#include -#include -#include namespace DB { +class ASTQueryParameter; + class QueryParameterVisitor { public: - QueryParameterVisitor(NameSet & parameters_name) : query_parameters(parameters_name) {} + QueryParameterVisitor(NameSet & parameters_name); - void visit(const ASTPtr & ast) - { - for (const auto & child : ast->children) - { - if (const auto & query_parameter = child->as()) - visitQueryParameter(*query_parameter); - else - visit(child); - } - } + void visit(const ASTPtr & ast); private: NameSet & query_parameters; - void visitQueryParameter(const ASTQueryParameter & query_parameter) - { - query_parameters.insert(query_parameter.name); - } + void visitQueryParameter(const ASTQueryParameter & query_parameter); }; -NameSet analyzeReceiveQueryParams(const std::string & query) -{ - NameSet query_params; - const char * query_begin = query.data(); - const char * query_end = query.data() + query.size(); - - ParserQuery parser(query_end, false); - ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "analyzeReceiveQueryParams", 0, 0); - QueryParameterVisitor(query_params).visit(extract_query_ast); - return query_params; -} +NameSet analyzeReceiveQueryParams(const std::string & query); } diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 5f520505a8a..bd16843517f 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -126,6 +126,7 @@ SRCS( QueryAliasesVisitor.cpp QueryLog.cpp QueryNormalizer.cpp + QueryParameterVisitor.cpp QueryThreadLog.cpp RemoveInjectiveFunctionsVisitor.cpp RenameColumnVisitor.cpp From c3040327073dabc711dd22aaba6527b566f67250 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 18:02:11 +0300 Subject: [PATCH 181/390] Even more --- src/Interpreters/QueryParameterVisitor.cpp | 43 +++++++++++++--------- src/Interpreters/QueryParameterVisitor.h | 19 ++-------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/Interpreters/QueryParameterVisitor.cpp b/src/Interpreters/QueryParameterVisitor.cpp index 8a257830443..ccf51ff41ce 100644 --- a/src/Interpreters/QueryParameterVisitor.cpp +++ b/src/Interpreters/QueryParameterVisitor.cpp @@ -1,6 +1,5 @@ -#pragma once - #include +#include #include #include #include @@ -9,26 +8,36 @@ namespace DB { -QueryParameterVisitor::QueryParameterVisitor(NameSet & parameters_name) - : query_parameters(parameters_name) -{ -} +class ASTQueryParameter; -void QueryParameterVisitor::visit(const ASTPtr & ast) +class QueryParameterVisitor { - for (const auto & child : ast->children) +public: + QueryParameterVisitor(NameSet & parameters_name) + : query_parameters(parameters_name) { - if (const auto & query_parameter = child->as()) - visitQueryParameter(*query_parameter); - else - visit(child); } -} -void QueryParameterVisitor::visitQueryParameter(const ASTQueryParameter & query_parameter) -{ - query_parameters.insert(query_parameter.name); -} + void visit(const ASTPtr & ast) + { + for (const auto & child : ast->children) + { + if (const auto & query_parameter = child->as()) + visitQueryParameter(*query_parameter); + else + visit(child); + } + } + +private: + NameSet & query_parameters; + + void visitQueryParameter(const ASTQueryParameter & query_parameter) + { + query_parameters.insert(query_parameter.name); + } +}; + NameSet analyzeReceiveQueryParams(const std::string & query) { diff --git a/src/Interpreters/QueryParameterVisitor.h b/src/Interpreters/QueryParameterVisitor.h index 24ef5b08dcf..531de2ddafa 100644 --- a/src/Interpreters/QueryParameterVisitor.h +++ b/src/Interpreters/QueryParameterVisitor.h @@ -1,26 +1,13 @@ #pragma once +#include #include -#include + namespace DB { -class ASTQueryParameter; - -class QueryParameterVisitor -{ -public: - QueryParameterVisitor(NameSet & parameters_name); - - void visit(const ASTPtr & ast); - -private: - NameSet & query_parameters; - - void visitQueryParameter(const ASTQueryParameter & query_parameter); -}; - +/// Find parameters in a query and collect them into set. NameSet analyzeReceiveQueryParams(const std::string & query); } From b87ecc2ec182ecb3fbda9347c8af0aedb97f5e34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 18:09:12 +0300 Subject: [PATCH 182/390] Even more --- src/Interpreters/QueryParameterVisitor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/QueryParameterVisitor.cpp b/src/Interpreters/QueryParameterVisitor.cpp index ccf51ff41ce..d96209e640a 100644 --- a/src/Interpreters/QueryParameterVisitor.cpp +++ b/src/Interpreters/QueryParameterVisitor.cpp @@ -8,8 +8,6 @@ namespace DB { -class ASTQueryParameter; - class QueryParameterVisitor { public: From c7a98132ac3e3ed336ebbc7acd93ee2f5222b50a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 18:13:51 +0300 Subject: [PATCH 183/390] Slightly better code --- src/Interpreters/ReplaceQueryParameterVisitor.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index ec824ed0cce..3dabfb06770 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -1,13 +1,10 @@ -#include #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -63,11 +60,14 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) data_type->deserializeAsTextEscaped(temp_column, read_buffer, format_settings); if (!read_buffer.eof()) - throw Exception("Value " + value + " cannot be parsed as " + type_name + " for query parameter '" + ast_param.name + "'" - " because it isn't parsed completely: only " + toString(read_buffer.count()) + " of " + toString(value.size()) + " bytes was parsed: " - + value.substr(0, read_buffer.count()), ErrorCodes::BAD_QUERY_PARAMETER); + throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, + "Value {} cannot be parsed as {} for query parameter '{}'" + " because it isn't parsed completely: only {} of {} bytes was parsed: {}", + value, type_name, ast_param.name, read_buffer.count(), value.size(), value.substr(0, read_buffer.count())); ast = addTypeConversionToAST(std::make_shared(temp_column[0]), type_name); + + /// Keep the original alias. ast->setAlias(alias); } From 03aec83aec5847e5187d94d312a626930399951c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 19:27:37 +0300 Subject: [PATCH 184/390] Remove obsolete code from performance test --- docker/test/performance-comparison/perf.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index ab360ceb04f..8b1c4d623dc 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -103,18 +103,6 @@ if not args.long: print('skipped\tTest is tagged as long.') sys.exit(0) -# Check main metric to detect infinite tests. We shouldn't have such tests anymore, -# but we did in the past, and it is convenient to be able to process old tests. -main_metric_element = root.find('main_metric/*') -if main_metric_element is not None and main_metric_element.tag != 'min_time': - raise Exception('Only the min_time main metric is supported. This test uses \'{}\''.format(main_metric_element.tag)) - -# Another way to detect infinite tests. They should have an appropriate main_metric -# but sometimes they don't. -infinite_sign = root.find('.//average_speed_not_changing_for_ms') -if infinite_sign is not None: - raise Exception('Looks like the test is infinite (sign 1)') - # Print report threshold for the test if it is set. if 'max_ignored_relative_change' in root.attrib: print(f'report-threshold\t{root.attrib["max_ignored_relative_change"]}') From 9fa49a68433beda485c08cceafe0cc395753d430 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 19:29:15 +0300 Subject: [PATCH 185/390] Fix clang-tidy --- src/Interpreters/QueryParameterVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/QueryParameterVisitor.cpp b/src/Interpreters/QueryParameterVisitor.cpp index d96209e640a..297bc211712 100644 --- a/src/Interpreters/QueryParameterVisitor.cpp +++ b/src/Interpreters/QueryParameterVisitor.cpp @@ -11,7 +11,7 @@ namespace DB class QueryParameterVisitor { public: - QueryParameterVisitor(NameSet & parameters_name) + explicit QueryParameterVisitor(NameSet & parameters_name) : query_parameters(parameters_name) { } From 7ae4d0174406a70e6d3a19de0bf96d92e2e84fa4 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 19 Sep 2020 19:31:08 +0300 Subject: [PATCH 186/390] Update convertMySQLDataType.cpp --- src/DataTypes/convertMySQLDataType.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/convertMySQLDataType.cpp b/src/DataTypes/convertMySQLDataType.cpp index ef6fb7de874..c67f90e6408 100644 --- a/src/DataTypes/convertMySQLDataType.cpp +++ b/src/DataTypes/convertMySQLDataType.cpp @@ -105,11 +105,10 @@ DataTypePtr convertMySQLDataType(MultiEnum type_support, else if (precision <= DecimalUtils::maxPrecision()) res = std::make_shared>(precision, scale); } - else - { - /// Also String is fallback for all unknown types. + + /// Also String is fallback for all unknown types. + if (!res) res = std::make_shared(); - } if (is_nullable) res = std::make_shared(res); From c78d9f9f385b5c3e4f6c8a87c85f3c8ba5b6423a Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 19:42:36 +0300 Subject: [PATCH 187/390] moved the generators to docs/ folder --- CMakeLists.txt | 14 +- cmake/find/ccache.cmake | 3 +- cmake/find/gtest.cmake | 7 +- cmake/sanitize.cmake | 2 +- cmake/tools.cmake | 2 +- cmake_flags_and_output.md | 276 ------------------ .../_includes/cmake_in_clickhouse_footer.md | 0 .../_includes/cmake_in_clickhouse_header.md | 2 +- docs/en/development/cmake_in_clickhouse.md | 275 +++++++++++++++++ docs/tools/build.py | 3 + .../tools/cmake_in_clickhouse_generator.py | 71 +++-- programs/CMakeLists.txt | 4 +- src/Functions/CMakeLists.txt | 6 +- src/Functions/TargetSpecific.h | 2 +- 14 files changed, 343 insertions(+), 324 deletions(-) delete mode 100644 cmake_flags_and_output.md rename cmake_files_footer.md => docs/_includes/cmake_in_clickhouse_footer.md (100%) rename cmake_files_header.md => docs/_includes/cmake_in_clickhouse_header.md (91%) create mode 100644 docs/en/development/cmake_in_clickhouse.md rename cmake_flags_generator.py => docs/tools/cmake_in_clickhouse_generator.py (66%) mode change 100755 => 100644 diff --git a/CMakeLists.txt b/CMakeLists.txt index fbd6b921289..3eacb57809a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,9 +59,9 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# Need cmake 3.9+ +# cmake 3.9+ needed. # Usually impractical. -# See also ENABLE_THINLTO +# See also ${ENABLE_THINLTO} option(ENABLE_IPO "Full link time optimization") if(ENABLE_IPO) @@ -150,7 +150,8 @@ if (COMPILER_CLANG) endif () endif () -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" ON) +# If turned `ON`, assumes the user has either the system GTest library or the bundled one. +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") # Only for Linux, x86_64. @@ -188,7 +189,8 @@ else () set(NO_WHOLE_ARCHIVE --no-whole-archive) endif () -option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used.") +# Ignored if `lld` is used +option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.") if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") if (LINKER_NAME STREQUAL "lld") @@ -211,8 +213,10 @@ endif() cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd + if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000) - option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON) + # Less `/tmp` usage, more RAM usage. + option(COMPILER_PIPE "-pipe compiler option" ON) endif() if(COMPILER_PIPE) diff --git a/cmake/find/ccache.cmake b/cmake/find/ccache.cmake index 270db1b4e66..8e9fe4d84ce 100644 --- a/cmake/find/ccache.cmake +++ b/cmake/find/ccache.cmake @@ -18,7 +18,8 @@ if (NOT CCACHE_FOUND AND NOT DEFINED ENABLE_CCACHE AND NOT COMPILER_MATCHES_CCAC "Setting it up will significantly reduce compilation time for 2nd and consequent builds") endif() -option(ENABLE_CCACHE "Speedup re-compilations using ccache" ${ENABLE_CCACHE_BY_DEFAULT}) +# https://ccache.dev/ +option(ENABLE_CCACHE "Speedup re-compilations using ccache (external tool)" ${ENABLE_CCACHE_BY_DEFAULT}) if (NOT ENABLE_CCACHE) return() diff --git a/cmake/find/gtest.cmake b/cmake/find/gtest.cmake index 36e45a1381e..ce2a73620b5 100644 --- a/cmake/find/gtest.cmake +++ b/cmake/find/gtest.cmake @@ -1,8 +1,6 @@ -option (ENABLE_GTEST_LIBRARY "Enable gtest library" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_GTEST_LIBRARY) +if (NOT ENABLE_TESTS) if(USE_INTERNAL_GTEST_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal Google Test when ENABLE_GTEST_LIBRARY=OFF") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal Google Test when ENABLE_TESTS=OFF") endif() return() endif() @@ -15,6 +13,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeList message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gtest") set (USE_INTERNAL_GTEST_LIBRARY 0) endif () + set (MISSING_INTERNAL_GTEST_LIBRARY 1) endif () diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 68cc4693ca0..0ccd6933dec 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -1,4 +1,4 @@ -# Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) +# Possible values: `address` (ASan), `memory` (MSan), `thread` (TSan), `undefined` (UBSan), and "" (no sanitizing) option (SANITIZE "Enable one of the code sanitizers" "") set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 734da46a8df..6f07cc2439c 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -40,7 +40,7 @@ endif () STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) -# Example values: lld-10, gold +# Example values: `lld-10`, `gold`. option (LINKER_NAME "Linker name or full path") if (COMPILER_GCC AND NOT LINKER_NAME) diff --git a/cmake_flags_and_output.md b/cmake_flags_and_output.md deleted file mode 100644 index 033cdf3e943..00000000000 --- a/cmake_flags_and_output.md +++ /dev/null @@ -1,276 +0,0 @@ -# CMake in ClickHouse - -## TL; DR How to make ClickHouse compile and link faster? - -Developer only! This command will likely fulfill most of your needs. Run before calling `ninja`. - -```cmake -cmake .. \ - -DCMAKE_C_COMPILER=/bin/clang-10 \ - -DCMAKE_CXX_COMPILER=/bin/clang++-10 \ - -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_CLICKHOUSE_ALL=OFF \ - -DENABLE_CLICKHOUSE_SERVER=ON \ - -DENABLE_CLICKHOUSE_CLIENT=ON \ - -DUSE_STATIC_LIBRARIES=OFF \ - -DCLICKHOUSE_SPLIT_BINARY=ON \ - -DSPLIT_SHARED_LIBRARIES=ON \ - -DENABLE_LIBRARIES=OFF \ - -DENABLE_UTILS=OFF \ - -DENABLE_TESTS=OFF -``` - -## CMake files types - -1. ClickHouse's source CMake files (located in the root directory and in `/src`). -2. Arch-dependent CMake files (located in `/cmake/*os_name*`). -3. Libraries finders (search for contrib libraries, located in `/cmake/find`). -3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) - -## List of CMake flags - -* This list is auto-generated by [this bash script](bash.sh). -* The flag name is a link to its position in the code. -* If an option's default value is itself an option, it's also a link to its position in this list. -### ClickHouse modes - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Ddata compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | -### External libraries - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | -| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gtest library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | -| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -### External libraries system/bundled mode - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | -| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | -| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | -| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | -| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | -| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | -| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | -| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | -| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | -| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | -| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | -| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | -| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | -| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | -| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | -### Other flags - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L190) | `OFF` | Add .gdb-index to resulting binaries for gold linker. NOOP if lld is used. | | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L227) | `OFF` | Add -march=native compiler flag | | -| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L214) | `ON` | -pipe compiler option [less /tmp usage, more ram usage] | | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies ${WITH_COVERAGE} | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | Need cmake 3.9+ Usually impractical. See also ENABLE_THINLTO | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L329) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | -| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L115) | `ON` | | | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L152) | `ON` | Provide unit_test_dbms target with Google.test unit tests | | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | Need cmake 3.9+ Usually impractical. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L157) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies ${ENABLE_FASTMEMCPY} | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: lld-10, gold | -| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: address (ASan), memory (MSan), thread (TSan), undefined (UBSan), and "" (no sanitizing) | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L154) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L390) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | -| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L345) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | -| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | - -## Developer's guide for adding new CMake options - -### Don't be obvious. Be informative. - -Bad: -```cmake -option (ENABLE_TESTS "Enables testing" OFF) -``` - -This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. - -Better: - -```cmake -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some -pre-conditions, leave a comment above the `option()` line and explain what it does. -The best way would be linking the docs page (if it exists). -The comment is parsed into a separate column (see below). - -Even better: - -```cmake -# implies ${TESTS_ARE_ENABLED} -# see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. - -Suppose you have an option that may strip debug symbols from the ClickHouse's part. -This can speed up the linking process, but produces a binary that cannot be debugged. -In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. -Also, such options should be disabled if applies. - -Bad: -```cmake -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() - -``` -Better: - -```cmake -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions." - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -### In the option's description, explain WHAT the option does rather than WHY it does something. - -The WHY explanation should be placed in the comment. -You may find that the option's name is self-descriptive. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better: - -```cmake -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -### Don't assume other developers know as much as you do. - -In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to -the tool's docs. It won't take much of your time. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better (combined with the above hint): - -```cmake -# https://clang.llvm.org/docs/ThinLTO.html -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -Other example, bad: - -```cmake -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) -``` - -Better: - -```cmake -# https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) -``` - -### Prefer consistent default values. - -CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. -Prefer the `ON/OFF` values, if possible. diff --git a/cmake_files_footer.md b/docs/_includes/cmake_in_clickhouse_footer.md similarity index 100% rename from cmake_files_footer.md rename to docs/_includes/cmake_in_clickhouse_footer.md diff --git a/cmake_files_header.md b/docs/_includes/cmake_in_clickhouse_header.md similarity index 91% rename from cmake_files_header.md rename to docs/_includes/cmake_in_clickhouse_header.md index 669399e9996..9d4cd0fb2e9 100644 --- a/cmake_files_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -29,6 +29,6 @@ cmake .. \ ## List of CMake flags -* This list is auto-generated by [this bash script](bash.sh). +* This list is auto-generated by [this Python script](docs/tools/cmake_in_clickhouse_generator.py). * The flag name is a link to its position in the code. * If an option's default value is itself an option, it's also a link to its position in this list. diff --git a/docs/en/development/cmake_in_clickhouse.md b/docs/en/development/cmake_in_clickhouse.md new file mode 100644 index 00000000000..585786f4645 --- /dev/null +++ b/docs/en/development/cmake_in_clickhouse.md @@ -0,0 +1,275 @@ +# CMake in ClickHouse + +## TL; DR How to make ClickHouse compile and link faster? + +Developer only! This command will likely fulfill most of your needs. Run before calling `ninja`. + +```cmake +cmake .. \ + -DCMAKE_C_COMPILER=/bin/clang-10 \ + -DCMAKE_CXX_COMPILER=/bin/clang++-10 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DENABLE_CLICKHOUSE_ALL=OFF \ + -DENABLE_CLICKHOUSE_SERVER=ON \ + -DENABLE_CLICKHOUSE_CLIENT=ON \ + -DUSE_STATIC_LIBRARIES=OFF \ + -DCLICKHOUSE_SPLIT_BINARY=ON \ + -DSPLIT_SHARED_LIBRARIES=ON \ + -DENABLE_LIBRARIES=OFF \ + -DENABLE_UTILS=OFF \ + -DENABLE_TESTS=OFF +``` + +## CMake files types + +1. ClickHouse's source CMake files (located in the root directory and in `/src`). +2. Arch-dependent CMake files (located in `/cmake/*os_name*`). +3. Libraries finders (search for contrib libraries, located in `/cmake/find`). +3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) + +## List of CMake flags + +* This list is auto-generated by [this Python script](docs/tools/cmake_in_clickhouse_generator.py). +* The flag name is a link to its position in the code. +* If an option's default value is itself an option, it's also a link to its position in this list. +### ClickHouse modes + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Data compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | +### External libraries + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.### External libraries system/bundled mode + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | +| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | +| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | +| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | +| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | +| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gtest.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | +| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | +| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | +| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | +| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | +| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | +| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | +| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | +| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | +| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | +| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | +| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | +| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | +### Other flags + +| Name | Default value | Description | Comment | +|------|---------------|-------------|---------| +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L192) | `OFF` | Add .gdb-index to resulting binaries for gold linker. | Ignored if `lld` is used | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | +| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | +| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/../../src/Functions/CMakeLists.txt#L115) | `ON` | Enable platform-dependent code | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies `ENABLE_FASTMEMCPY` | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: `lld-10`, `gold`. | +| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | +| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: `address` (ASan), `memory` (MSan), `thread` (TSan), `undefined` (UBSan), and "" (no sanitizing) | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/../../src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L394) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | +| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L349) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | +| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | + +## Developer's guide for adding new CMake options + +### Don't be obvious. Be informative. + +Bad: +```cmake +option (ENABLE_TESTS "Enables testing" OFF) +``` + +This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. + +Better: + +```cmake +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) +``` + +If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some +pre-conditions, leave a comment above the `option()` line and explain what it does. +The best way would be linking the docs page (if it exists). +The comment is parsed into a separate column (see below). + +Even better: + +```cmake +# implies ${TESTS_ARE_ENABLED} +# see tests/CMakeLists.txt for implementation detail. +option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) +``` + +### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. + +Suppose you have an option that may strip debug symbols from the ClickHouse's part. +This can speed up the linking process, but produces a binary that cannot be debugged. +In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. +Also, such options should be disabled if applies. + +Bad: +```cmake +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions. + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() + +``` +Better: + +```cmake +# Provides faster linking and lower binary size. +# Tradeoff is the inability to debug some source files with e.g. gdb +# (empty stack frames and no local variables)." +option(STRIP_DEBUG_SYMBOLS_FUNCTIONS + "Do not generate debugger info for ClickHouse functions." + ${STRIP_DSF_DEFAULT}) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + message(WARNING "Not generating debugger info for ClickHouse functions") + target_compile_options(clickhouse_functions PRIVATE "-g0") +endif() +``` + +### In the option's description, explain WHAT the option does rather than WHY it does something. + +The WHY explanation should be placed in the comment. +You may find that the option's name is self-descriptive. + +Bad: + +```cmake +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better: + +```cmake +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). +``` + +### Don't assume other developers know as much as you do. + +In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to +the tool's docs. It won't take much of your time. + +Bad: + +```cmake +option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) +``` + +Better (combined with the above hint): + +```cmake +# https://clang.llvm.org/docs/ThinLTO.html +# Only applicable for clang. +# Turned off when building with tests or sanitizers. +option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). +``` + +Other example, bad: + +```cmake +option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) +``` + +Better: + +```cmake +# https://github.com/include-what-you-use/include-what-you-use +option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) +``` + +### Prefer consistent default values. + +CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. +Prefer the `ON/OFF` values, if possible. diff --git a/docs/tools/build.py b/docs/tools/build.py index 120af33c8fb..52051da663c 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -28,6 +28,7 @@ import test import util import website +from cmake_in_clickhouse_generator import generate_cmake_flags_files class ClickHouseMarkdown(markdown.extensions.Extension): class ClickHousePreprocessor(markdown.util.Processor): @@ -184,6 +185,8 @@ def build(args): test.test_templates(args.website_dir) if not args.skip_docs: + generate_cmake_flags_files(root_path="../../") + build_docs(args) from github import build_releases build_releases(args, build_docs) diff --git a/cmake_flags_generator.py b/docs/tools/cmake_in_clickhouse_generator.py old mode 100755 new mode 100644 similarity index 66% rename from cmake_flags_generator.py rename to docs/tools/cmake_in_clickhouse_generator.py index 2c79595b564..822a76a816b --- a/cmake_flags_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -2,20 +2,24 @@ import re import os.path from typing import TextIO, List, Tuple, Optional, Dict +# name, default value, description Entity = Tuple[str, str, str] -# https://regex101.com/r/R6iogw/11 +# https://regex101.com/r/R6iogw/12 cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" -output_file_name: str = "cmake_flags_and_output.md" -header_file_name: str = "cmake_files_header.md" -footer_file_name: str = "cmake_files_footer.md" +output_file_name: str = "../en/development/cmake_in_clickhouse.md" +header_file_name: str = "../_includes/cmake_in_clickhouse_header.md" +footer_file_name: str = "../_includes/cmake_in_clickhouse_footer.md" ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" default_anchor_str: str = "[`{name}`](#{anchor})" +comment_var_regex: str = r"\${(.+)}" +comment_var_replace: str = "`\\1`" + table_header: str = """ | Name | Default value | Description | Comment | |------|---------------|-------------|---------| @@ -29,37 +33,42 @@ entities: Dict[str, Tuple[str, str]] = {} def make_anchor(t: str) -> str: return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) -def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str], **options) -> None: - (line, comment) = line_comment - (_name, _description, default) = entity +def process_comment(comment: str) -> str: + return re.sub(comment_var_regex, comment_var_replace, comment, flags=re.MULTILINE) - if _name in entities: +def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> None: + (line, comment) = line_comment + (name, description, default) = entity + + if name in entities: return if len(default) == 0: - default = "`OFF`" + formatted_default: str = "`OFF`" elif default[0] == "$": - default = default[2:-1] - default = default_anchor_str.format( - name=default, - anchor=make_anchor(default)) + formatted_default: str = default[2:-1] + formatted_default: str = default_anchor_str.format( + name=formatted_default, + anchor=make_anchor(formatted_default)) else: - default = "`" + default + "`" + formatted_default: str = "`" + default + "`" - name: str = name_str.format( - anchor=make_anchor(_name), - name=_name, + formatted_name: str = name_str.format( + anchor=make_anchor(name), + name=name, path=path, line=line if line > 0 else 1) - if options.get("no_desc", False): - description: str = "" - else: - description: str = "".join(_description.split("\n")) + " | " + formatted_description: str = "".join(description.split("\n")) - entities[_name] = path, "| " + name + " | " + default + " | " + description + comment + " |" + formatted_comment: str = process_comment(comment) -def process_file(input_name: str, **options) -> None: + formatted_entity: str = "| {} | {} | {} | {} |".format( + formatted_name, formatted_default, formatted_description, formatted_comment) + + entities[name] = path, formatted_entity + +def process_file(input_name: str) -> None: with open(input_name, 'r') as cmake_file: contents: str = cmake_file.read() @@ -91,13 +100,13 @@ def process_folder(name: str) -> None: if f == "CMakeLists.txt" or ".cmake" in f: process_file(root + "/" + f) -def process() -> None: - process_file("CMakeLists.txt") - process_file("programs/CMakeLists.txt", no_desc=True) +def generate_cmake_flags_files(root_path: str) -> None: + process_file(root_path + "CMakeLists.txt") + process_file(root_path + "programs/CMakeLists.txt") - process_folder("base") - process_folder("cmake") - process_folder("src") + process_folder(root_path + "base") + process_folder(root_path + "cmake") + process_folder(root_path + "src") with open(output_file_name, "w") as f: with open(header_file_name, "r") as header: @@ -115,6 +124,8 @@ def process() -> None: f.write("### External libraries\n" + table_header) + f.write("Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.") + for k in sorted_keys: if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): f.write(entities[k][1] + "\n") @@ -134,5 +145,3 @@ def process() -> None: with open(footer_file_name, "r") as footer: f.write(footer.read()) - -process() diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index be2f1ca5ba5..3577ee3df31 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,7 +2,7 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -# The 'clickhouse' binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), +# The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. # If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. option (ENABLE_CLICKHOUSE_ALL "Enable all ClickHouse modes by default" ON) @@ -20,7 +20,7 @@ option (ENABLE_CLICKHOUSE_BENCHMARK "Queries benchmarking mode" ${ENABLE_CLICKHO option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Configs processor (extract values etc.)" ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ -option (ENABLE_CLICKHOUSE_COMPRESSOR "Ddata compressor and decompressor" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_COMPRESSOR "Data compressor and decompressor" ${ENABLE_CLICKHOUSE_ALL}) # https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ option (ENABLE_CLICKHOUSE_COPIER "Inter-cluster data copying mode" ${ENABLE_CLICKHOUSE_ALL}) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index fd37e14685f..eb8404a5ad6 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -113,7 +113,11 @@ if(USE_RAPIDJSON) target_include_directories(clickhouse_functions SYSTEM PRIVATE ${RAPIDJSON_INCLUDE_DIR}) endif() -option(ENABLE_MULTITARGET_CODE "" ON) +# ClickHouse developers may use platform-dependent code under some macro (e.g. `#ifdef ENABLE_MULTITARGET_CODE`). +# If turned ON, this option defines such macro. +# See `src/Functions/TargetSpecific.h` +option(ENABLE_MULTITARGET_CODE "Enable platform-dependent code" ON) + if (ENABLE_MULTITARGET_CODE) add_definitions(-DENABLE_MULTITARGET_CODE=1) else() diff --git a/src/Functions/TargetSpecific.h b/src/Functions/TargetSpecific.h index bc433702180..8de6a3dbec4 100644 --- a/src/Functions/TargetSpecific.h +++ b/src/Functions/TargetSpecific.h @@ -11,7 +11,7 @@ * * If compiler is not gcc/clang or target isn't x86_64 or ENABLE_MULTITARGET_CODE * was set to OFF in cmake, all code inside these macros will be removed and - * USE_MUTLITARGE_CODE will be set to 0. Use #if USE_MUTLITARGE_CODE whenever you + * USE_MULTITARGET_CODE will be set to 0. Use #if USE_MULTITARGET_CODE whenever you * use anything from this namespaces. * * For similarities there is a macros DECLARE_DEFAULT_CODE, which wraps code From 194c8da69a302f5cff694ea22d84cacc1db53fa3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 19:49:24 +0300 Subject: [PATCH 188/390] Tweak test --- ...ference => 01502_long_log_tinylog_deadlock_race.reference} | 0 ...adlock_race.sh => 01502_long_log_tinylog_deadlock_race.sh} | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/queries/0_stateless/{01502_log_tinylog_deadlock_race.reference => 01502_long_log_tinylog_deadlock_race.reference} (100%) rename tests/queries/0_stateless/{01502_log_tinylog_deadlock_race.sh => 01502_long_log_tinylog_deadlock_race.sh} (89%) diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference similarity index 100% rename from tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference rename to tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh similarity index 89% rename from tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh rename to tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh index a5b2ff6db8f..29c5f868617 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh @@ -10,14 +10,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function thread_create { while true; do - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' sleep 0.0$RANDOM done } function thread_drop { while true; do - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' sleep 0.0$RANDOM done } From 2544d1442ddbdd09e44fba2d485408d158453ed6 Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 19:51:21 +0300 Subject: [PATCH 189/390] replaced _ with - in file name --- ...n_clickhouse.md => cmake-in-clickhouse.md} | 63 ++++++++++--------- docs/tools/cmake_in_clickhouse_generator.py | 11 ++-- 2 files changed, 38 insertions(+), 36 deletions(-) rename docs/en/development/{cmake_in_clickhouse.md => cmake-in-clickhouse.md} (99%) diff --git a/docs/en/development/cmake_in_clickhouse.md b/docs/en/development/cmake-in-clickhouse.md similarity index 99% rename from docs/en/development/cmake_in_clickhouse.md rename to docs/en/development/cmake-in-clickhouse.md index 585786f4645..12e0e15c6b8 100644 --- a/docs/en/development/cmake_in_clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -49,10 +49,41 @@ cmake .. \ | [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | | [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | ### External libraries +Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras. | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.### External libraries system/bundled mode +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | +| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | +### External libraries system/bundled mode | Name | Default value | Description | Comment | |------|---------------|-------------|---------| @@ -95,40 +126,10 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click | [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | | [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | | [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | -| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | | [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | | [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | -| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | | [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/../../src/Functions/CMakeLists.txt#L115) | `ON` | Enable platform-dependent code | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | | [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | | [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | | [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 822a76a816b..6fd81f87403 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -8,7 +8,7 @@ Entity = Tuple[str, str, str] # https://regex101.com/r/R6iogw/12 cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" -output_file_name: str = "../en/development/cmake_in_clickhouse.md" +output_file_name: str = "../en/development/cmake-in-clickhouse.md" header_file_name: str = "../_includes/cmake_in_clickhouse_header.md" footer_file_name: str = "../_includes/cmake_in_clickhouse_footer.md" @@ -122,12 +122,11 @@ def generate_cmake_flags_files(root_path: str) -> None: f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("### External libraries\n" + table_header) - - f.write("Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.") + f.write("### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" + + table_header) for k in sorted_keys: - if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): + if k.startswith("ENABLE_") and entities[k][0].startswith(root_path + "cmake"): f.write(entities[k][1] + "\n") ignored_keys.append(k) @@ -145,3 +144,5 @@ def generate_cmake_flags_files(root_path: str) -> None: with open(footer_file_name, "r") as footer: f.write(footer.read()) + +generate_cmake_flags_files("../../") From 146dd0f3f00c64cd3586ec2b8db5b3259a1e2602 Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 20:14:37 +0300 Subject: [PATCH 190/390] fixed comments parser --- docs/en/development/cmake-in-clickhouse.md | 208 ++++++++++---------- docs/tools/cmake_in_clickhouse_generator.py | 22 +-- src/Functions/CMakeLists.txt | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md index 12e0e15c6b8..dadae1a7b62 100644 --- a/docs/en/development/cmake-in-clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -36,123 +36,123 @@ cmake .. \ | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Data compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/../../programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | +| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | +| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | +| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | +| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Data compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | +| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | +| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | +| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | +| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | +| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | +| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | +| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | +| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | ### External libraries Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras. | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | -| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | -| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | ### External libraries system/bundled mode | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | -| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | -| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | -| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | -| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | -| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/gtest.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | -| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | -| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | -| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | -| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | -| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | -| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | -| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | -| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | -| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | +| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | +| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | +| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | +| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | +| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | +| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | +| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | +| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | +| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | +| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | +| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | +| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | +| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | +| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | +| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | +| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | +| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | +| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | +| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | +| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | +| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | +| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | +| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | +| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | +| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | +| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | +| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | +| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | +| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | +| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | ### Other flags | Name | Default value | Description | Comment | |------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L192) | `OFF` | Add .gdb-index to resulting binaries for gold linker. | Ignored if `lld` is used | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | -| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | -| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/../../src/Functions/CMakeLists.txt#L115) | `ON` | Enable platform-dependent code | | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies `ENABLE_FASTMEMCPY` | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: `lld-10`, `gold`. | -| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: `address` (ASan), `memory` (MSan), `thread` (TSan), `undefined` (UBSan), and "" (no sanitizing) | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/../../src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L394) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | -| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L349) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | -| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/../../cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/../../CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | +| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L192) | `OFF` | Add .gdb-index to resulting binaries for gold linker. | Ignored if `lld` is used | +| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | +| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | +| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | +| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | +| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | +| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | +| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L118) | `ON` | Enable platform-dependent code | ClickHouse developers may use platform-dependent code under some macro (e.g. `ifdef ENABLE_MULTITARGET`). If turned ON, this option defines such macro. See `src/Functions/TargetSpecific.h` | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | +| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | +| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | +| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | +| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies `ENABLE_FASTMEMCPY` | +| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: `lld-10`, `gold`. | +| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | +| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | +| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | +| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | +| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: `address` (ASan), `memory` (MSan), `thread` (TSan), `undefined` (UBSan), and "" (no sanitizing) | +| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | +| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | +| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | +| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L394) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | +| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | +| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | +| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | +| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | +| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | +| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | +| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L349) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | +| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | +| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | ## Developer's guide for adding new CMake options diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 6fd81f87403..20a90b96b0f 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -1,5 +1,5 @@ import re -import os.path +import os from typing import TextIO, List, Tuple, Optional, Dict # name, default value, description @@ -8,9 +8,9 @@ Entity = Tuple[str, str, str] # https://regex101.com/r/R6iogw/12 cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" -output_file_name: str = "../en/development/cmake-in-clickhouse.md" -header_file_name: str = "../_includes/cmake_in_clickhouse_header.md" -footer_file_name: str = "../_includes/cmake_in_clickhouse_footer.md" +output_file_name: str = "docs/en/development/cmake-in-clickhouse.md" +header_file_name: str = "docs/_includes/cmake_in_clickhouse_header.md" +footer_file_name: str = "docs/_includes/cmake_in_clickhouse_footer.md" ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" @@ -101,12 +101,14 @@ def process_folder(name: str) -> None: process_file(root + "/" + f) def generate_cmake_flags_files(root_path: str) -> None: - process_file(root_path + "CMakeLists.txt") - process_file(root_path + "programs/CMakeLists.txt") + os.chdir(root_path) - process_folder(root_path + "base") - process_folder(root_path + "cmake") - process_folder(root_path + "src") + process_file("CMakeLists.txt") + process_file("programs/CMakeLists.txt") + + process_folder("base") + process_folder("cmake") + process_folder("src") with open(output_file_name, "w") as f: with open(header_file_name, "r") as header: @@ -144,5 +146,3 @@ def generate_cmake_flags_files(root_path: str) -> None: with open(footer_file_name, "r") as footer: f.write(footer.read()) - -generate_cmake_flags_files("../../") diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index eb8404a5ad6..bdf89c983f1 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -113,7 +113,7 @@ if(USE_RAPIDJSON) target_include_directories(clickhouse_functions SYSTEM PRIVATE ${RAPIDJSON_INCLUDE_DIR}) endif() -# ClickHouse developers may use platform-dependent code under some macro (e.g. `#ifdef ENABLE_MULTITARGET_CODE`). +# ClickHouse developers may use platform-dependent code under some macro (e.g. `#ifdef ENABLE_MULTITARGET`). # If turned ON, this option defines such macro. # See `src/Functions/TargetSpecific.h` option(ENABLE_MULTITARGET_CODE "Enable platform-dependent code" ON) From af95fdfbefc05c8bff78adce02cef9bbafb8875c Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 20:28:40 +0300 Subject: [PATCH 191/390] fixed libraries parser --- docs/en/development/cmake-in-clickhouse.md | 60 ++++++++++----------- docs/tools/cmake_in_clickhouse_generator.py | 2 +- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md index dadae1a7b62..439625eab1c 100644 --- a/docs/en/development/cmake-in-clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -53,6 +53,36 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click | Name | Default value | Description | Comment | |------|---------------|-------------|---------| +| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | +| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | +| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | +| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | +| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | +| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | +| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | +| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | +| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | +| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | +| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | +| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | +| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | +| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | +| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | +| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | +| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | +| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | +| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | +| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | +| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | +| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | +| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | +| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | +| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | +| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | +| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | +| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | +| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | +| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | ### External libraries system/bundled mode | Name | Default value | Description | Comment | @@ -96,40 +126,10 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click | [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | | [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | | [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | -| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | | [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | | [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | -| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | | [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | | [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L118) | `ON` | Enable platform-dependent code | ClickHouse developers may use platform-dependent code under some macro (e.g. `ifdef ENABLE_MULTITARGET`). If turned ON, this option defines such macro. See `src/Functions/TargetSpecific.h` | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | | [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | | [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | | [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 20a90b96b0f..11ace80dbc2 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -128,7 +128,7 @@ def generate_cmake_flags_files(root_path: str) -> None: table_header) for k in sorted_keys: - if k.startswith("ENABLE_") and entities[k][0].startswith(root_path + "cmake"): + if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): f.write(entities[k][1] + "\n") ignored_keys.append(k) From 0afaca52f27c9a870ef0363bbf86386fa2a5a957 Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 20:31:41 +0300 Subject: [PATCH 192/390] finishing the PR --- docs/_includes/cmake_in_clickhouse_header.md | 2 +- docs/en/development/cmake-in-clickhouse.md | 276 ------------------- 2 files changed, 1 insertion(+), 277 deletions(-) delete mode 100644 docs/en/development/cmake-in-clickhouse.md diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index 9d4cd0fb2e9..10776e04c01 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -29,6 +29,6 @@ cmake .. \ ## List of CMake flags -* This list is auto-generated by [this Python script](docs/tools/cmake_in_clickhouse_generator.py). +* This list is auto-generated by [this Python script](https://github.com/clickhouse/clickhouse/blob/master/docs/tools/cmake_in_clickhouse_generator.py). * The flag name is a link to its position in the code. * If an option's default value is itself an option, it's also a link to its position in this list. diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md deleted file mode 100644 index 439625eab1c..00000000000 --- a/docs/en/development/cmake-in-clickhouse.md +++ /dev/null @@ -1,276 +0,0 @@ -# CMake in ClickHouse - -## TL; DR How to make ClickHouse compile and link faster? - -Developer only! This command will likely fulfill most of your needs. Run before calling `ninja`. - -```cmake -cmake .. \ - -DCMAKE_C_COMPILER=/bin/clang-10 \ - -DCMAKE_CXX_COMPILER=/bin/clang++-10 \ - -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_CLICKHOUSE_ALL=OFF \ - -DENABLE_CLICKHOUSE_SERVER=ON \ - -DENABLE_CLICKHOUSE_CLIENT=ON \ - -DUSE_STATIC_LIBRARIES=OFF \ - -DCLICKHOUSE_SPLIT_BINARY=ON \ - -DSPLIT_SHARED_LIBRARIES=ON \ - -DENABLE_LIBRARIES=OFF \ - -DENABLE_UTILS=OFF \ - -DENABLE_TESTS=OFF -``` - -## CMake files types - -1. ClickHouse's source CMake files (located in the root directory and in `/src`). -2. Arch-dependent CMake files (located in `/cmake/*os_name*`). -3. Libraries finders (search for contrib libraries, located in `/cmake/find`). -3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) - -## List of CMake flags - -* This list is auto-generated by [this Python script](docs/tools/cmake_in_clickhouse_generator.py). -* The flag name is a link to its position in the code. -* If an option's default value is itself an option, it's also a link to its position in this list. -### ClickHouse modes - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ENABLE_CLICKHOUSE_ALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L7) | `ON` | Enable all ClickHouse modes by default | The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. | -| [`ENABLE_CLICKHOUSE_BENCHMARK`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L17) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries benchmarking mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-benchmark/ | -| [`ENABLE_CLICKHOUSE_CLIENT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L10) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Client mode (interactive tui/shell that connects to the server) | | -| [`ENABLE_CLICKHOUSE_COMPRESSOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L22) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Data compressor and decompressor | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-compressor/ | -| [`ENABLE_CLICKHOUSE_COPIER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L25) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Inter-cluster data copying mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-copier/ | -| [`ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L19) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Configs processor (extract values etc.) | | -| [`ENABLE_CLICKHOUSE_FORMAT`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L27) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Queries pretty-printer and formatter with syntax highlighting | | -| [`ENABLE_CLICKHOUSE_INSTALL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L39) | `OFF` | Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only) | | -| [`ENABLE_CLICKHOUSE_LOCAL`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L14) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Local files fast processing mode | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-local/ | -| [`ENABLE_CLICKHOUSE_OBFUSCATOR`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L31) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Table data obfuscator (convert real data to benchmark-ready one) | https://clickhouse.tech/docs/en/operations/utilities/clickhouse-obfuscator/ | -| [`ENABLE_CLICKHOUSE_ODBC_BRIDGE`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L35) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | HTTP-server working like a proxy to ODBC driver | https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/ | -| [`ENABLE_CLICKHOUSE_SERVER`](https://github.com/clickhouse/clickhouse/blob/master/programs/CMakeLists.txt#L9) | [`ENABLE_CLICKHOUSE_ALL`](#enable-clickhouse-all) | Server mode (main mode) | | -### External libraries -Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras. - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ENABLE_AMQPCPP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/amqpcpp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe AMQP-CPP | | -| [`ENABLE_AVRO`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Avro | | -| [`ENABLE_BASE64`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/base64.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable base64 | | -| [`ENABLE_BROTLI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable brotli | | -| [`ENABLE_CAPNP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cap'n Proto | | -| [`ENABLE_CASSANDRA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cassandra.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable Cassandra | | -| [`ENABLE_CCACHE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ccache.cmake#L6) | [`ENABLE_CCACHE_BY_DEFAULT`](#enable-ccache-by-default) | Speedup re-compilations using ccache (external tool) | | -| [`ENABLE_CLANG_TIDY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/analysis.cmake#L1) | `OFF` | Use clang-tidy static analyzer | https://clang.llvm.org/extra/clang-tidy/ | -| [`ENABLE_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable curl | | -| [`ENABLE_EMBEDDED_COMPILER`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Set to TRUE to enable support for 'compile_expressions' option for query execution | | -| [`ENABLE_FASTOPS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/fastops.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable fast vectorized mathematical functions library by Mikhail Parakhin | | -| [`ENABLE_GPERF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gperf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gperf function hash generator tool | | -| [`ENABLE_GRPC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use gRPC | | -| [`ENABLE_GSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable gsasl library | | -| [`ENABLE_H3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable H3 | | -| [`ENABLE_HDFS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable HDFS | | -| [`ENABLE_ICU`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ICU | | -| [`ENABLE_LDAP`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable LDAP | | -| [`ENABLE_MSGPACK`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable msgpack library | | -| [`ENABLE_MYSQL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable MySQL | | -| [`ENABLE_ODBC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ODBC library | | -| [`ENABLE_OPENCL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/opencl.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable OpenCL support | | -| [`ENABLE_ORC`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ORC | | -| [`ENABLE_PARQUET`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable parquet | | -| [`ENABLE_PROTOBUF`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable protobuf | | -| [`ENABLE_RAPIDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use rapidjson | | -| [`ENABLE_RDKAFKA`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L2) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable kafka | | -| [`ENABLE_S3`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable S3 | | -| [`ENABLE_SSL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable ssl | | -| [`ENABLE_STATS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/stats.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enalbe StatsLib library | | -### External libraries system/bundled mode - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`USE_INTERNAL_${LIB_NAME_UC}_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/contrib_finder.cmake#L6) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled library ${LIB_NAME} instead of system | | -| [`USE_INTERNAL_AVRO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/avro.cmake#L3) | `ON` | Set to FALSE to use system avro library instead of bundled | | -| [`USE_INTERNAL_AWS_S3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/s3.cmake#L2) | `ON` | Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_BROTLI_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/brotli.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libbrotli library instead of bundled | | -| [`USE_INTERNAL_CAPNP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/capnp.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system capnproto library instead of bundled | | -| [`USE_INTERNAL_CURL`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/curl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal curl library | | -| [`USE_INTERNAL_GRPC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/grpc.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk) | | -| [`USE_INTERNAL_GTEST_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/gtest.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system Google Test instead of bundled | | -| [`USE_INTERNAL_H3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/h3.cmake#L2) | `ON` | Set to FALSE to use system h3 library instead of bundled | | -| [`USE_INTERNAL_HDFS3_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/hdfs3.cmake#L2) | `ON` | Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk) | | -| [`USE_INTERNAL_ICU_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/icu.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system ICU library instead of bundled | | -| [`USE_INTERNAL_LDAP_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ldap.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *LDAP library instead of bundled | | -| [`USE_INTERNAL_LIBCXX_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L3) | [`USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT`](#use-internal-libcxx-library-default) | Disable to use system libcxx and libcxxabi libraries instead of bundled | | -| [`USE_INTERNAL_LIBGSASL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libgsasl.cmake#L3) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Set to FALSE to use system libgsasl library instead of bundled | | -| [`USE_INTERNAL_LIBXML2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/libxml2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system libxml2 library instead of bundled | | -| [`USE_INTERNAL_LLVM_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Use bundled or system LLVM library. | | -| [`USE_INTERNAL_MSGPACK_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/msgpack.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system msgpack library instead of bundled | | -| [`USE_INTERNAL_MYSQL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/mysqlclient.cmake#L7) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system mysqlclient library instead of bundled | | -| [`USE_INTERNAL_ODBC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/odbc.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Use internal ODBC library | | -| [`USE_INTERNAL_ORC_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/orc.cmake#L3) | `ON` | Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk) | | -| [`USE_INTERNAL_PARQUET_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/parquet.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system parquet library instead of bundled | | -| [`USE_INTERNAL_POCO_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/poco.cmake#L1) | `ON` | Use internal Poco library | | -| [`USE_INTERNAL_PROTOBUF_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/protobuf.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system protobuf instead of bundled | | -| [`USE_INTERNAL_RAPIDJSON_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rapidjson.cmake#L2) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system rapidjson library instead of bundled | | -| [`USE_INTERNAL_RDKAFKA_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/rdkafka.cmake#L10) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system librdkafka instead of the bundled | | -| [`USE_INTERNAL_RE2_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/re2.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system re2 library instead of bundled [slower] | | -| [`USE_INTERNAL_SNAPPY_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system snappy library instead of bundled | | -| [`USE_INTERNAL_SPARSEHASH_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sparsehash.cmake#L1) | `ON` | Set to FALSE to use system sparsehash library instead of bundled | | -| [`USE_INTERNAL_SSL_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/ssl.cmake#L3) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system *ssl library instead of bundled | | -| [`USE_INTERNAL_ZLIB_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zlib.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zlib library instead of bundled | | -| [`USE_INTERNAL_ZSTD_LIBRARY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/zstd.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Set to FALSE to use system zstd library instead of bundled | | -### Other flags - -| Name | Default value | Description | Comment | -|------|---------------|-------------|---------| -| [`ADD_GDB_INDEX_FOR_GOLD`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L192) | `OFF` | Add .gdb-index to resulting binaries for gold linker. | Ignored if `lld` is used | -| [`ARCH_NATIVE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L231) | `OFF` | Add -march=native compiler flag | | -| [`CLICKHOUSE_SPLIT_BINARY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L103) | `OFF` | Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled | | -| [`COMPILER_PIPE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L218) | `ON` | -pipe compiler option | Less `/tmp` usage, more RAM usage. | -| [`ENABLE_FUZZING`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L120) | `OFF` | Fuzzy testing using libfuzzer | Implies `WITH_COVERAGE` | -| [`ENABLE_IPO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L64) | `OFF` | Full link time optimization | cmake 3.9+ needed. Usually impractical. See also `ENABLE_THINLTO` | -| [`ENABLE_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L333) | `ON` | Enable all external libraries by default | Turns on all external libs like s3, kafka, ODBC, ... | -| [`ENABLE_MULTITARGET_CODE`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L118) | `ON` | Enable platform-dependent code | ClickHouse developers may use platform-dependent code under some macro (e.g. `ifdef ENABLE_MULTITARGET`). If turned ON, this option defines such macro. See `src/Functions/TargetSpecific.h` | -| [`ENABLE_TESTS`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L153) | `ON` | Provide unit_test_dbms target with Google.Test unit tests | If turned `ON`, assumes the user has either the system GTest library or the bundled one. | -| [`ENABLE_THINLTO`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L63) | `ON` | Clang-specific link time optimization | cmake 3.9+ needed. Usually impractical. | -| [`FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L31) | `ON` | Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy | If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. | -| [`GLIBC_COMPATIBILITY`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L158) | `ON` | Enable compatibility with older glibc libraries. | Only for Linux, x86_64. Implies `ENABLE_FASTMEMCPY` | -| [`LINKER_NAME`](https://github.com/clickhouse/clickhouse/blob/master/cmake/tools.cmake#L43) | `OFF` | Linker name or full path | Example values: `lld-10`, `gold`. | -| [`LLVM_HAS_RTTI`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/llvm.cmake#L39) | `ON` | Enable if LLVM was build with RTTI enabled | | -| [`MAKE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L96) | [`USE_STATIC_LIBRARIES`](#use-static-libraries) | Disable to make shared libraries | | -| [`PARALLEL_COMPILE_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L9) | `""` | Maximum number of concurrent compilation jobs | 1 if not set | -| [`PARALLEL_LINK_JOBS`](https://github.com/clickhouse/clickhouse/blob/master/cmake/limit_jobs.cmake#L12) | `""` | Maximum number of concurrent link jobs | 1 if not set | -| [`SANITIZE`](https://github.com/clickhouse/clickhouse/blob/master/cmake/sanitize.cmake#L1) | `""` | Enable one of the code sanitizers | Possible values: `address` (ASan), `memory` (MSan), `thread` (TSan), `undefined` (UBSan), and "" (no sanitizing) | -| [`SPLIT_SHARED_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L101) | `OFF` | Keep all internal libraries as separate .so files | DEVELOPER ONLY. Faster linking if turned on. | -| [`STRIP_DEBUG_SYMBOLS_FUNCTIONS`](https://github.com/clickhouse/clickhouse/blob/master/src/Functions/CMakeLists.txt#L67) | [`STRIP_DSF_DEFAULT`](#strip-dsf-default) | Do not generate debugger info for ClickHouse functions | Provides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)." | -| [`UNBUNDLED`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L155) | `OFF` | Use system libraries instead of ones in contrib/ | | -| [`USE_INCLUDE_WHAT_YOU_USE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L394) | `OFF` | Automatically reduce unneeded includes in source code (external tool) | https://github.com/include-what-you-use/include-what-you-use | -| [`USE_LIBCXX`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/cxx.cmake#L1) | [`NOT_UNBUNDLED`](#not-unbundled) | Use libc++ and libc++abi instead of libstdc++ | | -| [`USE_SENTRY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/sentry.cmake#L5) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use Sentry | | -| [`USE_SIMDJSON`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/simdjson.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Use simdjson | | -| [`USE_SNAPPY`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/snappy.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable snappy library | | -| [`USE_STATIC_LIBRARIES`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L95) | `ON` | Disable to use shared libraries | | -| [`USE_UNWIND`](https://github.com/clickhouse/clickhouse/blob/master/cmake/find/unwind.cmake#L1) | [`ENABLE_LIBRARIES`](#enable-libraries) | Enable libunwind (better stacktraces) | | -| [`WERROR`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L349) | `OFF` | Enable -Werror compiler option | Using system libs can cause a lot of warnings in includes (on macro expansion). | -| [`WEVERYTHING`](https://github.com/clickhouse/clickhouse/blob/master/cmake/warnings.cmake#L21) | `ON` | Enable -Weverything option with some exceptions. | Add some warnings that are not available even with -Wall -Wextra -Wpedantic. Intended for exploration of new compiler warnings that may be found useful. Applies to clang only | -| [`WITH_COVERAGE`](https://github.com/clickhouse/clickhouse/blob/master/CMakeLists.txt#L119) | `OFF` | Profile the resulting binary/binaries | | - -## Developer's guide for adding new CMake options - -### Don't be obvious. Be informative. - -Bad: -```cmake -option (ENABLE_TESTS "Enables testing" OFF) -``` - -This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. - -Better: - -```cmake -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some -pre-conditions, leave a comment above the `option()` line and explain what it does. -The best way would be linking the docs page (if it exists). -The comment is parsed into a separate column (see below). - -Even better: - -```cmake -# implies ${TESTS_ARE_ENABLED} -# see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. - -Suppose you have an option that may strip debug symbols from the ClickHouse's part. -This can speed up the linking process, but produces a binary that cannot be debugged. -In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. -Also, such options should be disabled if applies. - -Bad: -```cmake -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() - -``` -Better: - -```cmake -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions." - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -### In the option's description, explain WHAT the option does rather than WHY it does something. - -The WHY explanation should be placed in the comment. -You may find that the option's name is self-descriptive. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better: - -```cmake -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -### Don't assume other developers know as much as you do. - -In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to -the tool's docs. It won't take much of your time. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better (combined with the above hint): - -```cmake -# https://clang.llvm.org/docs/ThinLTO.html -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -Other example, bad: - -```cmake -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) -``` - -Better: - -```cmake -# https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) -``` - -### Prefer consistent default values. - -CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. -Prefer the `ON/OFF` values, if possible. From 35e8a895b646c4aa7266e5beb98e000a8038a143 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 20:32:13 +0300 Subject: [PATCH 193/390] Calculate profiler data in separate query runs --- docker/test/performance-comparison/Dockerfile | 2 +- .../users.d/perf-comparison-tweaks-users.xml | 2 - docker/test/performance-comparison/perf.py | 46 ++++++++++++++++--- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index df666af8e8e..a4f8af2f388 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -29,7 +29,7 @@ RUN apt-get update \ tzdata \ vim \ wget \ - && pip3 --no-cache-dir install clickhouse_driver \ + && pip3 --no-cache-dir install clickhouse_driver scipy \ && apt-get purge --yes python3-dev g++ \ && apt-get autoremove --yes \ && apt-get clean \ diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml index c6d9f7ea582..cee7dc3ff16 100644 --- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml +++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml @@ -1,8 +1,6 @@ - 10000000 - 0 1 1 1 diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 8b1c4d623dc..cc55e1de5f9 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -15,6 +15,7 @@ import sys import time import traceback import xml.etree.ElementTree as et +from scipy import stats def tsv_escape(s): return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') @@ -211,7 +212,8 @@ for query_index in queries_to_run: for conn_index, c in enumerate(all_connections): try: prewarm_id = f'{query_prefix}.prewarm0' - res = c.execute(q, query_id = prewarm_id) + # Will also detect too long queries during warmup stage + res = c.execute(q, query_id = prewarm_id, settings = {'max_execution_time': 10}) print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') except KeyboardInterrupt: raise @@ -221,7 +223,6 @@ for query_index in queries_to_run: query_error_on_connection[conn_index] = traceback.format_exc(); continue - # Report all errors that ocurred during prewarm and decide what to do next. # If prewarm fails for the query on all servers -- skip the query and # continue testing the next query. @@ -246,7 +247,12 @@ for query_index in queries_to_run: # sending excessive data. start_seconds = time.perf_counter() server_seconds = 0 + profile_seconds = 0 run = 0 + + # Arrays of run times for each connection. + all_server_times = [[]] * len(this_query_connections) + while True: run_id = f'{query_prefix}.run{run}' @@ -259,14 +265,17 @@ for query_index in queries_to_run: e.message = run_id + ': ' + e.message raise - server_seconds += c.last_query.elapsed - print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}') + elapsed = c.last_query.elapsed + all_server_times[conn_index].append(elapsed) - if c.last_query.elapsed > 10: + server_seconds += elapsed + print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}') + + if elapsed > 10: # Stop processing pathologically slow queries, to avoid timing out # the entire test task. This shouldn't really happen, so we don't # need much handling for this case and can just exit. - print(f'The query no. {query_index} is taking too long to run ({c.last_query.elapsed} s)', file=sys.stderr) + print(f'The query no. {query_index} is taking too long to run ({c.elapsed} s)', file=sys.stderr) exit(2) # Be careful with the counter, after this line it's the next iteration @@ -297,6 +306,31 @@ for query_index in queries_to_run: client_seconds = time.perf_counter() - start_seconds print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}') + # Run additional profiling queries to collect profile data, but only if test times appeared to be different. + # We have to do it after normal runs because otherwise it will affect test statistics too much + if len(all_server_times) == 2 and stats.ttest_ind(all_server_times[0], all_server_times[1], equal_var = False).pvalue < 0.1: + while True: + run_id = f'{query_prefix}.profile{run}' + + for conn_index, c in enumerate(this_query_connections): + try: + res = c.execute(q, query_id = run_id, settings = {'query_profiler_real_time_period_ns': 10000000}) + except Exception as e: + # Add query id to the exception to make debugging easier. + e.args = (run_id, *e.args) + e.message = run_id + ': ' + e.message + raise + + elapsed = c.last_query.elapsed + profile_seconds += elapsed + + # Don't spend too much time for profile runs + if run >= args.runs or profile_seconds > 10: + continue + + run += 1 + # And don't bother with short queries + # Run drop queries drop_queries = substitute_parameters(drop_query_templates) for conn_index, c in enumerate(all_connections): From eb1bb88fa0c32d464ba1bd5d26c3ecce4edb9ded Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 20:45:11 +0300 Subject: [PATCH 194/390] updated the gtest find cmake file --- cmake/find/gtest.cmake | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmake/find/gtest.cmake b/cmake/find/gtest.cmake index ce2a73620b5..9d4ab2608cb 100644 --- a/cmake/find/gtest.cmake +++ b/cmake/find/gtest.cmake @@ -1,9 +1,4 @@ -if (NOT ENABLE_TESTS) - if(USE_INTERNAL_GTEST_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal Google Test when ENABLE_TESTS=OFF") - endif() - return() -endif() +# included only if ENABLE_TESTS=1 option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test instead of bundled" ${NOT_UNBUNDLED}) From 07317644eb50230f8119e43c160b58b568e6d79d Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 21:05:40 +0300 Subject: [PATCH 195/390] added explicit tests disabled status --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3eacb57809a..e19ec8ace33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -405,7 +405,9 @@ if (USE_INCLUDE_WHAT_YOU_USE) endif () if (ENABLE_TESTS) - message (STATUS "Tests are enabled") + message (STATUS "Unit tests are enabled") +else() + message(STATUS "Unit tests are disabled") endif () enable_testing() # Enable for tests without binary From ef800b508351108e59769a3587c5c1d0ff6dbab3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 21:21:29 +0300 Subject: [PATCH 196/390] Perf test: development --- docker/test/performance-comparison/perf.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index cc55e1de5f9..a37ef553cfb 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -251,7 +251,9 @@ for query_index in queries_to_run: run = 0 # Arrays of run times for each connection. - all_server_times = [[]] * len(this_query_connections) + all_server_times = [] + for conn_index, c in enumerate(this_query_connections): + all_server_times.append([]) while True: run_id = f'{query_prefix}.run{run}' @@ -306,15 +308,20 @@ for query_index in queries_to_run: client_seconds = time.perf_counter() - start_seconds print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}') + #print(all_server_times) + #print(stats.ttest_ind(all_server_times[0], all_server_times[1], equal_var = False).pvalue) + # Run additional profiling queries to collect profile data, but only if test times appeared to be different. # We have to do it after normal runs because otherwise it will affect test statistics too much if len(all_server_times) == 2 and stats.ttest_ind(all_server_times[0], all_server_times[1], equal_var = False).pvalue < 0.1: + run = 0 while True: run_id = f'{query_prefix}.profile{run}' for conn_index, c in enumerate(this_query_connections): try: res = c.execute(q, query_id = run_id, settings = {'query_profiler_real_time_period_ns': 10000000}) + print(f'profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}') except Exception as e: # Add query id to the exception to make debugging easier. e.args = (run_id, *e.args) @@ -324,11 +331,10 @@ for query_index in queries_to_run: elapsed = c.last_query.elapsed profile_seconds += elapsed - # Don't spend too much time for profile runs - if run >= args.runs or profile_seconds > 10: - continue - run += 1 + # Don't spend too much time for profile runs + if run > args.runs or profile_seconds > 10: + break # And don't bother with short queries # Run drop queries From aba66a53b168f0bb9a3fa75eec6d7aa55784122b Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 21:41:38 +0300 Subject: [PATCH 197/390] replace os.chdir with relative path --- docs/tools/cmake_in_clickhouse_generator.py | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 11ace80dbc2..d3aff8c5894 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -8,10 +8,6 @@ Entity = Tuple[str, str, str] # https://regex101.com/r/R6iogw/12 cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" -output_file_name: str = "docs/en/development/cmake-in-clickhouse.md" -header_file_name: str = "docs/_includes/cmake_in_clickhouse_header.md" -footer_file_name: str = "docs/_includes/cmake_in_clickhouse_footer.md" - ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" @@ -68,8 +64,8 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No entities[name] = path, formatted_entity -def process_file(input_name: str) -> None: - with open(input_name, 'r') as cmake_file: +def process_file(root_path: str, input_name: str) -> None: + with open(root_path + input_name, 'r') as cmake_file: contents: str = cmake_file.read() def get_line_and_comment(target: str) -> Tuple[int, str]: @@ -94,21 +90,23 @@ def process_file(input_name: str) -> None: for entity in matches: build_entity(input_name, entity, get_line_and_comment(entity[0])) -def process_folder(name: str) -> None: +def process_folder(root_path:str, name: str) -> None: for root, _, files in os.walk(name): for f in files: if f == "CMakeLists.txt" or ".cmake" in f: - process_file(root + "/" + f) + process_file(root_path, root + "/" + f) def generate_cmake_flags_files(root_path: str) -> None: - os.chdir(root_path) + output_file_name: str = root_path + "docs/en/development/cmake-in-clickhouse.md" + header_file_name: str = root_path + "docs/_includes/cmake_in_clickhouse_header.md" + footer_file_name: str = root_path + "docs/_includes/cmake_in_clickhouse_footer.md" - process_file("CMakeLists.txt") - process_file("programs/CMakeLists.txt") + process_file(root_path, "CMakeLists.txt") + process_file(root_path, "programs/CMakeLists.txt") - process_folder("base") - process_folder("cmake") - process_folder("src") + process_folder(root_path, "base") + process_folder(root_path, "cmake") + process_folder(root_path, "src") with open(output_file_name, "w") as f: with open(header_file_name, "r") as header: @@ -146,3 +144,6 @@ def generate_cmake_flags_files(root_path: str) -> None: with open(footer_file_name, "r") as footer: f.write(footer.read()) + + +generate_cmake_flags_files("../../") From 8bda21f9f1d812fb5425eb8eadd0f7266e4affeb Mon Sep 17 00:00:00 2001 From: myrrc Date: Sat, 19 Sep 2020 21:45:10 +0300 Subject: [PATCH 198/390] added conditional execution ability --- docs/tools/cmake_in_clickhouse_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index d3aff8c5894..9ac6e9475d8 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -146,4 +146,5 @@ def generate_cmake_flags_files(root_path: str) -> None: f.write(footer.read()) -generate_cmake_flags_files("../../") +if __name__ == '__main__': + generate_cmake_flags_files("../../") From dae3b40a55ef9bb7ea47692ec882fb8a55677ce6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 22:15:16 +0300 Subject: [PATCH 199/390] Fix buffer overflow in "bar" function --- src/Common/UnicodeBar.cpp | 70 +++++++++++++++++++++++++++++++++++++++ src/Common/UnicodeBar.h | 52 +++-------------------------- src/Common/ya.make | 1 + 3 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 src/Common/UnicodeBar.cpp diff --git a/src/Common/UnicodeBar.cpp b/src/Common/UnicodeBar.cpp new file mode 100644 index 00000000000..8ff5e2052c1 --- /dev/null +++ b/src/Common/UnicodeBar.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + namespace ErrorCodes + { + extern const int PARAMETER_OUT_OF_BOUND; + } +} + + +namespace UnicodeBar +{ + double getWidth(Int64 x, Int64 min, Int64 max, double max_width) + { + if (x <= min) + return 0; + + if (x >= max) + return max_width; + + /// The case when max - min overflows + Int64 max_difference; + if (common::subOverflow(max, min, max_difference)) + throw DB::Exception(DB::ErrorCodes::PARAMETER_OUT_OF_BOUND, "The arguments to render unicode bar will lead to arithmetic overflow"); + + return (x - min) * max_width / max_difference; + } + + size_t getWidthInBytes(double width) + { + return ceil(width - 1.0 / 8) * UNICODE_BAR_CHAR_SIZE; + } + + void render(double width, char * dst) + { + size_t floor_width = floor(width); + + for (size_t i = 0; i < floor_width; ++i) + { + memcpy(dst, "█", UNICODE_BAR_CHAR_SIZE); + dst += UNICODE_BAR_CHAR_SIZE; + } + + size_t remainder = floor((width - floor_width) * 8); + + if (remainder) + { + memcpy(dst, &"▏▎▍▌▋▋▊▉"[(remainder - 1) * UNICODE_BAR_CHAR_SIZE], UNICODE_BAR_CHAR_SIZE); + dst += UNICODE_BAR_CHAR_SIZE; + } + + *dst = 0; + } + + std::string render(double width) + { + std::string res(getWidthInBytes(width), '\0'); + render(width, res.data()); + return res; + } +} + diff --git a/src/Common/UnicodeBar.h b/src/Common/UnicodeBar.h index 9a5bcecbd62..0c62bd7e8f7 100644 --- a/src/Common/UnicodeBar.h +++ b/src/Common/UnicodeBar.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include @@ -10,54 +8,12 @@ /** Allows you to draw a unicode-art bar whose width is displayed with a resolution of 1/8 character. */ - - namespace UnicodeBar { - using DB::Int64; - - inline double getWidth(Int64 x, Int64 min, Int64 max, double max_width) - { - if (x <= min) - return 0; - - if (x >= max) - return max_width; - - return (x - min) * max_width / (max - min); - } - - inline size_t getWidthInBytes(double width) - { - return ceil(width - 1.0 / 8) * UNICODE_BAR_CHAR_SIZE; - } + double getWidth(Int64 x, Int64 min, Int64 max, double max_width); + size_t getWidthInBytes(double width); /// In `dst` there must be a space for barWidthInBytes(width) characters and a trailing zero. - inline void render(double width, char * dst) - { - size_t floor_width = floor(width); - - for (size_t i = 0; i < floor_width; ++i) - { - memcpy(dst, "█", UNICODE_BAR_CHAR_SIZE); - dst += UNICODE_BAR_CHAR_SIZE; - } - - size_t remainder = floor((width - floor_width) * 8); - - if (remainder) - { - memcpy(dst, &"▏▎▍▌▋▋▊▉"[(remainder - 1) * UNICODE_BAR_CHAR_SIZE], UNICODE_BAR_CHAR_SIZE); - dst += UNICODE_BAR_CHAR_SIZE; - } - - *dst = 0; - } - - inline std::string render(double width) - { - std::string res(getWidthInBytes(width), '\0'); - render(width, res.data()); - return res; - } + void render(double width, char * dst); + std::string render(double width); } diff --git a/src/Common/ya.make b/src/Common/ya.make index 9b671e7b9ee..fb04ecaa141 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -99,6 +99,7 @@ SRCS( ThreadProfileEvents.cpp ThreadStatus.cpp TraceCollector.cpp + UnicodeBar.cpp UTF8Helpers.cpp WeakHash.cpp ZooKeeper/IKeeper.cpp From ace6130585c53643fe11bc87d2c423b5671e5faf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 22:28:43 +0300 Subject: [PATCH 200/390] Added a test --- tests/queries/0_stateless/01502_bar_overflow.reference | 0 tests/queries/0_stateless/01502_bar_overflow.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/01502_bar_overflow.reference create mode 100644 tests/queries/0_stateless/01502_bar_overflow.sql diff --git a/tests/queries/0_stateless/01502_bar_overflow.reference b/tests/queries/0_stateless/01502_bar_overflow.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01502_bar_overflow.sql b/tests/queries/0_stateless/01502_bar_overflow.sql new file mode 100644 index 00000000000..cb3de7ac20b --- /dev/null +++ b/tests/queries/0_stateless/01502_bar_overflow.sql @@ -0,0 +1 @@ +SELECT bar((greatCircleAngle(100, -1, number, number) - number) * 2, -9223372036854775808, 1023, 100) FROM numbers(1048575); -- { serverError 12 } From 5a47e64685075ea50e5f11b00bda5735e8fd88bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 22:34:24 +0300 Subject: [PATCH 201/390] Proper exception message for wrong number of arguments of CAST --- src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp | 3 +++ .../queries/0_stateless/01503_if_const_optimization.reference | 0 tests/queries/0_stateless/01503_if_const_optimization.sql | 1 + 3 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/01503_if_const_optimization.reference create mode 100644 tests/queries/0_stateless/01503_if_const_optimization.sql diff --git a/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp b/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp index dd63093493f..765e7b1fa3d 100644 --- a/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp +++ b/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp @@ -33,6 +33,9 @@ static bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & v { if (const auto * expr_list = function->arguments->as()) { + if (expr_list->children.size() != 2) + throw Exception("Function CAST must have exactly two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + const ASTPtr & type_ast = expr_list->children.at(1); if (const auto * type_literal = type_ast->as()) { diff --git a/tests/queries/0_stateless/01503_if_const_optimization.reference b/tests/queries/0_stateless/01503_if_const_optimization.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01503_if_const_optimization.sql b/tests/queries/0_stateless/01503_if_const_optimization.sql new file mode 100644 index 00000000000..047f6f757e8 --- /dev/null +++ b/tests/queries/0_stateless/01503_if_const_optimization.sql @@ -0,0 +1 @@ +SELECT if(CAST(NULL), '2.55', NULL) AS x; -- { serverError 42 } From 0029c5512d03d8cebedb3e9a07f350e60255e5e7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 22:43:26 +0300 Subject: [PATCH 202/390] Maybe fix MSan report in base64 --- src/Functions/FunctionBase64Conversion.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index 450bad496cd..b6217f9b0bc 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -4,6 +4,7 @@ #if USE_BASE64 # include +# include # include # include # include @@ -151,6 +152,10 @@ public: } } + /// Base64 library is using AVX-512 with some shuffle operations. + /// Memory sanitizer don't understand if there was uninitialized memory in SIMD register but it was not used in the result of shuffle. + __msan_unpoison(dst_pos, outlen); + source += srclen + 1; dst_pos += outlen + 1; From 71568130d8fe84c2b54d8411dc8918d969fdc814 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sat, 19 Sep 2020 18:33:02 -0300 Subject: [PATCH 203/390] more tests for #14646 --- .../01475_mutation_with_if.reference | 17 ++++++++ .../0_stateless/01475_mutation_with_if.sql | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tests/queries/0_stateless/01475_mutation_with_if.reference b/tests/queries/0_stateless/01475_mutation_with_if.reference index 2874a18147f..fed47882f5c 100644 --- a/tests/queries/0_stateless/01475_mutation_with_if.reference +++ b/tests/queries/0_stateless/01475_mutation_with_if.reference @@ -1 +1,18 @@ 1 150 +2020-02-28 car +2020-03-28 dog +2020-03-28 dog +2020-08-02 car +\N cat +2020-08-03 car +2020-03-28 dog +2020-08-02 car +2020-03-28 dog +2020-08-04 cat +2020-08-04 car +2020-08-04 car +\N cat +2020-08-05 \N +\N car +\N dog +\N car diff --git a/tests/queries/0_stateless/01475_mutation_with_if.sql b/tests/queries/0_stateless/01475_mutation_with_if.sql index 6f0ef8924be..e2207846e03 100644 --- a/tests/queries/0_stateless/01475_mutation_with_if.sql +++ b/tests/queries/0_stateless/01475_mutation_with_if.sql @@ -14,3 +14,42 @@ ALTER TABLE mutation_table UPDATE price = 150 WHERE id = 1 SETTINGS mutations_sy SELECT * FROM mutation_table; DROP TABLE IF EXISTS mutation_table; + + +create table mutation_table ( dt Nullable(Date), name Nullable(String)) +engine MergeTree order by tuple(); + +insert into mutation_table (name, dt) values ('car', '2020-02-28'); +insert into mutation_table (name, dt) values ('dog', '2020-03-28'); + +select * from mutation_table; + +alter table mutation_table update dt = toDateOrNull('2020-08-02') +where name = 'car' SETTINGS mutations_sync = 2; + +select * from mutation_table; + +insert into mutation_table (name, dt) values ('car', Null); +insert into mutation_table (name, dt) values ('cat', Null); + +alter table mutation_table update dt = toDateOrNull('2020-08-03') +where name = 'car' and dt is null SETTINGS mutations_sync = 2; + +select * from mutation_table; + +alter table mutation_table update dt = toDateOrNull('2020-08-04') +where name = 'car' or dt is null SETTINGS mutations_sync = 2; + +select * from mutation_table; + +insert into mutation_table (name, dt) values (Null, '2020-08-05'); + +alter table mutation_table update dt = Null +where name is not null SETTINGS mutations_sync = 2; + +select * from mutation_table; + + +DROP TABLE IF EXISTS mutation_table; + + From cd53a7016333368ea6f31e03bdfcf13aa41cc52e Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sat, 19 Sep 2020 18:40:14 -0300 Subject: [PATCH 204/390] constant output order --- .../0_stateless/01475_mutation_with_if.reference | 14 +++++++------- .../queries/0_stateless/01475_mutation_with_if.sql | 13 ++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/01475_mutation_with_if.reference b/tests/queries/0_stateless/01475_mutation_with_if.reference index fed47882f5c..2d6f3e8097a 100644 --- a/tests/queries/0_stateless/01475_mutation_with_if.reference +++ b/tests/queries/0_stateless/01475_mutation_with_if.reference @@ -3,16 +3,16 @@ 2020-03-28 dog 2020-03-28 dog 2020-08-02 car -\N cat -2020-08-03 car 2020-03-28 dog 2020-08-02 car -2020-03-28 dog -2020-08-04 cat -2020-08-04 car -2020-08-04 car +2020-08-03 car \N cat +2020-03-28 dog +2020-08-04 car +2020-08-04 car +2020-08-04 cat 2020-08-05 \N \N car -\N dog \N car +\N cat +\N dog diff --git a/tests/queries/0_stateless/01475_mutation_with_if.sql b/tests/queries/0_stateless/01475_mutation_with_if.sql index e2207846e03..c25d208e924 100644 --- a/tests/queries/0_stateless/01475_mutation_with_if.sql +++ b/tests/queries/0_stateless/01475_mutation_with_if.sql @@ -16,18 +16,19 @@ SELECT * FROM mutation_table; DROP TABLE IF EXISTS mutation_table; + create table mutation_table ( dt Nullable(Date), name Nullable(String)) engine MergeTree order by tuple(); insert into mutation_table (name, dt) values ('car', '2020-02-28'); insert into mutation_table (name, dt) values ('dog', '2020-03-28'); -select * from mutation_table; +select * from mutation_table order by dt, name; alter table mutation_table update dt = toDateOrNull('2020-08-02') where name = 'car' SETTINGS mutations_sync = 2; -select * from mutation_table; +select * from mutation_table order by dt, name; insert into mutation_table (name, dt) values ('car', Null); insert into mutation_table (name, dt) values ('cat', Null); @@ -35,21 +36,19 @@ insert into mutation_table (name, dt) values ('cat', Null); alter table mutation_table update dt = toDateOrNull('2020-08-03') where name = 'car' and dt is null SETTINGS mutations_sync = 2; -select * from mutation_table; +select * from mutation_table order by dt, name; alter table mutation_table update dt = toDateOrNull('2020-08-04') where name = 'car' or dt is null SETTINGS mutations_sync = 2; -select * from mutation_table; +select * from mutation_table order by dt, name; insert into mutation_table (name, dt) values (Null, '2020-08-05'); alter table mutation_table update dt = Null where name is not null SETTINGS mutations_sync = 2; -select * from mutation_table; +select * from mutation_table order by dt, name; DROP TABLE IF EXISTS mutation_table; - - From 80949fa48e666c67775346a42bfd1d26cde253e0 Mon Sep 17 00:00:00 2001 From: myrrc Date: Sun, 20 Sep 2020 01:27:36 +0300 Subject: [PATCH 205/390] an attempt to fix docs build --- docs/tools/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 52051da663c..c91cc8d5f3c 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -185,7 +185,7 @@ def build(args): test.test_templates(args.website_dir) if not args.skip_docs: - generate_cmake_flags_files(root_path="../../") + generate_cmake_flags_files(os.path.join(os.path.dirname(__file__), '..', '..')) build_docs(args) from github import build_releases @@ -203,6 +203,7 @@ def build(args): if __name__ == '__main__': os.chdir(os.path.join(os.path.dirname(__file__), '..')) website_dir = os.path.join('..', 'website') + arg_parser = argparse.ArgumentParser() arg_parser.add_argument('--lang', default='en,es,fr,ru,zh,ja,tr,fa') arg_parser.add_argument('--blog-lang', default='en,ru') From fcee786320da9f76b4b31bac1b93dd4a88398826 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 20 Sep 2020 11:53:29 +0800 Subject: [PATCH 206/390] Fix incorrect key condition of fixed strings. --- src/Interpreters/convertFieldToType.cpp | 13 +++++++++++++ .../01503_fixed_string_primary_key.reference | 1 + .../0_stateless/01503_fixed_string_primary_key.sql | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/01503_fixed_string_primary_key.reference create mode 100644 tests/queries/0_stateless/01503_fixed_string_primary_key.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 2d624922d2a..c0e9d7edc13 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -191,7 +191,20 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID else if (which_type.isStringOrFixedString()) { if (src.getType() == Field::Types::String) + { + if (which_type.isFixedString()) + { + size_t n = assert_cast(type).getN(); + const auto & src_str = src.get(); + if (src_str.size() < n) + { + String src_str_extended = src_str; + src_str_extended.resize(n); + return src_str_extended; + } + } return src; + } } else if (const DataTypeArray * type_array = typeid_cast(&type)) { diff --git a/tests/queries/0_stateless/01503_fixed_string_primary_key.reference b/tests/queries/0_stateless/01503_fixed_string_primary_key.reference new file mode 100644 index 00000000000..45a4fb75db8 --- /dev/null +++ b/tests/queries/0_stateless/01503_fixed_string_primary_key.reference @@ -0,0 +1 @@ +8 diff --git a/tests/queries/0_stateless/01503_fixed_string_primary_key.sql b/tests/queries/0_stateless/01503_fixed_string_primary_key.sql new file mode 100644 index 00000000000..09576777cb6 --- /dev/null +++ b/tests/queries/0_stateless/01503_fixed_string_primary_key.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test(key FixedString(10)) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY (key); +INSERT INTO test SELECT toString(intDiv(number, 8)) FROM numbers(100); +SELECT count() FROM test WHERE key = '1'; + +DROP TABLE IF EXISTS test; From 8ca4d6634eea9557b9fd8562ba94ac5840a92e8d Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 20 Sep 2020 15:16:46 +0800 Subject: [PATCH 207/390] better fixed string group by support --- src/Interpreters/Aggregator.cpp | 16 ++++++++-------- src/Interpreters/Aggregator.h | 8 ++++---- .../performance/single_fixed_string_groupby.xml | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 tests/performance/single_fixed_string_groupby.xml diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 86a33dccb53..1df76f96663 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -367,6 +367,14 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() throw Exception("Logical error: numeric column has sizeOfField not in 1, 2, 4, 8, 16, 32.", ErrorCodes::LOGICAL_ERROR); } + if (params.keys_size == 1 && isFixedString(types_removed_nullable[0])) + { + if (has_low_cardinality) + return AggregatedDataVariants::Type::low_cardinality_key_fixed_string; + else + return AggregatedDataVariants::Type::key_fixed_string; + } + /// If all keys fits in N bits, will use hash table with all keys packed (placed contiguously) to single N-bit key. if (params.keys_size == num_fixed_contiguous_keys) { @@ -399,14 +407,6 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() return AggregatedDataVariants::Type::key_string; } - if (params.keys_size == 1 && isFixedString(types_removed_nullable[0])) - { - if (has_low_cardinality) - return AggregatedDataVariants::Type::low_cardinality_key_fixed_string; - else - return AggregatedDataVariants::Type::key_fixed_string; - } - return AggregatedDataVariants::Type::serialized; } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 8dce011c435..fac7889d6fc 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -228,7 +228,7 @@ struct AggregationMethodString static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(key.data, key.size); + static_cast(key_columns[0].get())->insertData(key.data, key.size); } }; @@ -254,7 +254,7 @@ struct AggregationMethodStringNoCache static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(key.data, key.size); + static_cast(key_columns[0].get())->insertData(key.data, key.size); } }; @@ -280,7 +280,7 @@ struct AggregationMethodFixedString static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(key.data, key.size); + static_cast(key_columns[0].get())->insertData(key.data, key.size); } }; @@ -305,7 +305,7 @@ struct AggregationMethodFixedStringNoCache static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { - key_columns[0]->insertData(key.data, key.size); + static_cast(key_columns[0].get())->insertData(key.data, key.size); } }; diff --git a/tests/performance/single_fixed_string_groupby.xml b/tests/performance/single_fixed_string_groupby.xml new file mode 100644 index 00000000000..10ca5f1ebcd --- /dev/null +++ b/tests/performance/single_fixed_string_groupby.xml @@ -0,0 +1,17 @@ + + DROP TABLE IF EXISTS perf_lc_fixed_str_groupby + CREATE TABLE perf_lc_fixed_str_groupby( + a LowCardinality(FixedString(14)), + b LowCardinality(FixedString(14)) + ) ENGINE MergeTree ORDER BY tuple() + + + + INSERT INTO perf_lc_fixed_str_groupby SELECT ('number key ' || toString(number % 400)) AS a, ('number key ' || toString(number % 20)) AS b FROM numbers(30000000) + + + SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY a + SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY b + + DROP TABLE IF EXISTS perf_lc_fixed_str_groupby + From 5ffee8808a2c53c0528cd75d0128ffa24f368ff1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 20 Sep 2020 16:27:33 +0300 Subject: [PATCH 208/390] Alter remove column properties and TTLs (#14742) --- src/Databases/DatabaseOrdinary.cpp | 2 + src/Interpreters/InterpreterAlterQuery.cpp | 5 + src/Parsers/ASTAlterQuery.cpp | 21 ++- src/Parsers/ASTAlterQuery.h | 4 + src/Parsers/ParserAlterQuery.cpp | 46 ++++- src/Parsers/ParserCreateQuery.h | 28 ++- src/Storages/AlterCommands.cpp | 177 ++++++++++++++++-- src/Storages/AlterCommands.h | 31 ++- src/Storages/StorageReplicatedMergeTree.cpp | 22 ++- ...ter_remove_no_property_zookeeper.reference | 4 + ...493_alter_remove_no_property_zookeeper.sql | 51 +++++ .../01493_alter_remove_properties.reference | 20 ++ .../01493_alter_remove_properties.sql | 72 +++++++ ...lter_remove_properties_zookeeper.reference | 21 +++ ...1493_alter_remove_properties_zookeeper.sql | 92 +++++++++ ...01493_alter_remove_wrong_default.reference | 1 + .../01493_alter_remove_wrong_default.sql | 22 +++ 17 files changed, 570 insertions(+), 49 deletions(-) create mode 100644 tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.reference create mode 100644 tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.sql create mode 100644 tests/queries/0_stateless/01493_alter_remove_properties.reference create mode 100644 tests/queries/0_stateless/01493_alter_remove_properties.sql create mode 100644 tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.reference create mode 100644 tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql create mode 100644 tests/queries/0_stateless/01493_alter_remove_wrong_default.reference create mode 100644 tests/queries/0_stateless/01493_alter_remove_wrong_default.sql diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 0512a155418..13aeb7de148 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -291,6 +291,8 @@ void DatabaseOrdinary::alterTable(const Context & context, const StorageID & tab if (metadata.table_ttl.definition_ast) storage_ast.set(storage_ast.ttl_table, metadata.table_ttl.definition_ast); + else if (storage_ast.ttl_table != nullptr) /// TTL was removed + storage_ast.ttl_table = nullptr; if (metadata.settings_changes) storage_ast.set(storage_ast.settings, metadata.settings_changes); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 8cf581eb463..73251cad991 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -222,6 +222,11 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS required_access.emplace_back(AccessType::ALTER_TTL, database, table); break; } + case ASTAlterCommand::REMOVE_TTL: + { + required_access.emplace_back(AccessType::ALTER_TTL, database, table); + break; + } case ASTAlterCommand::MATERIALIZE_TTL: { required_access.emplace_back(AccessType::ALTER_MATERIALIZE_TTL, database, table); diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index d033cdc79a2..d74156d11d8 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -99,12 +99,19 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY COLUMN " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); col_decl->formatImpl(settings, state, frame); - if (first) - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " FIRST " << (settings.hilite ? hilite_none : ""); - else if (column) /// AFTER + if (!remove_property.empty()) { - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " AFTER " << (settings.hilite ? hilite_none : ""); - column->formatImpl(settings, state, frame); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " REMOVE " << remove_property; + } + else + { + if (first) + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " FIRST " << (settings.hilite ? hilite_none : ""); + else if (column) /// AFTER + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << " AFTER " << (settings.hilite ? hilite_none : ""); + column->formatImpl(settings, state, frame); + } } } else if (type == ASTAlterCommand::COMMENT_COLUMN) @@ -280,6 +287,10 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY TTL " << (settings.hilite ? hilite_none : ""); ttl->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::REMOVE_TTL) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REMOVE TTL" << (settings.hilite ? hilite_none : ""); + } else if (type == ASTAlterCommand::MATERIALIZE_TTL) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MATERIALIZE TTL" diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index df27ba0a3b0..78e0c726ddf 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -36,6 +36,7 @@ public: MATERIALIZE_TTL, MODIFY_SETTING, MODIFY_QUERY, + REMOVE_TTL, ADD_INDEX, DROP_INDEX, @@ -167,6 +168,9 @@ public: /// Target column name ASTPtr rename_to; + /// Which property user want to remove + String remove_property; + String getID(char delim) const override { return "AlterCommand" + (delim + std::to_string(static_cast(type))); } ASTPtr clone() const override; diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 9930bb649b4..0739f24a688 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -82,12 +82,23 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_where("WHERE"); ParserKeyword s_to("TO"); + ParserKeyword s_remove("REMOVE"); + ParserKeyword s_default("DEFAULT"); + ParserKeyword s_materialized("MATERIALIZED"); + ParserKeyword s_alias("ALIAS"); + ParserKeyword s_comment("COMMENT"); + ParserKeyword s_codec("CODEC"); + ParserKeyword s_ttl("TTL"); + + ParserKeyword s_remove_ttl("REMOVE TTL"); + ParserCompoundIdentifier parser_name; ParserStringLiteral parser_string_literal; + ParserIdentifier parser_remove_property; ParserCompoundColumnDeclaration parser_col_decl; ParserIndexDeclaration parser_idx_decl; ParserConstraintDeclaration parser_constraint_decl; - ParserCompoundColumnDeclaration parser_modify_col_decl(false); + ParserCompoundColumnDeclaration parser_modify_col_decl(false, false, true); ParserPartition parser_partition; ParserExpression parser_exp_elem; ParserList parser_assignment_list( @@ -433,14 +444,33 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!parser_modify_col_decl.parse(pos, command->col_decl, expected)) return false; - if (s_first.ignore(pos, expected)) - command->first = true; - else if (s_after.ignore(pos, expected)) + if (s_remove.ignore(pos, expected)) { - if (!parser_name.parse(pos, command->column, expected)) + if (s_default.ignore(pos, expected)) + command->remove_property = "DEFAULT"; + else if (s_materialized.ignore(pos, expected)) + command->remove_property = "MATERIALIZED"; + else if (s_alias.ignore(pos, expected)) + command->remove_property = "ALIAS"; + else if (s_comment.ignore(pos, expected)) + command->remove_property = "COMMENT"; + else if (s_codec.ignore(pos, expected)) + command->remove_property = "CODEC"; + else if (s_ttl.ignore(pos, expected)) + command->remove_property = "TTL"; + else return false; } - + else + { + if (s_first.ignore(pos, expected)) + command->first = true; + else if (s_after.ignore(pos, expected)) + { + if (!parser_name.parse(pos, command->column, expected)) + return false; + } + } command->type = ASTAlterCommand::MODIFY_COLUMN; } else if (s_modify_order_by.ignore(pos, expected)) @@ -496,6 +526,10 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected return false; command->type = ASTAlterCommand::MODIFY_TTL; } + else if (s_remove_ttl.ignore(pos, expected)) + { + command->type = ASTAlterCommand::REMOVE_TTL; + } else if (s_materialize_ttl.ignore(pos, expected)) { command->type = ASTAlterCommand::MATERIALIZE_TTL; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 953f657a04e..60933f7384d 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -90,8 +90,10 @@ template class IParserColumnDeclaration : public IParserBase { public: - explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false) - : require_type(require_type_), allow_null_modifiers(allow_null_modifiers_) + explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false, bool check_keywords_after_name_ = false) + : require_type(require_type_) + , allow_null_modifiers(allow_null_modifiers_) + , check_keywords_after_name(check_keywords_after_name_) { } @@ -104,6 +106,7 @@ protected: bool require_type = true; bool allow_null_modifiers = false; + bool check_keywords_after_name = false; }; using ParserColumnDeclaration = IParserColumnDeclaration; @@ -122,6 +125,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserKeyword s_comment{"COMMENT"}; ParserKeyword s_codec{"CODEC"}; ParserKeyword s_ttl{"TTL"}; + ParserKeyword s_remove{"REMOVE"}; ParserTernaryOperatorExpression expr_parser; ParserStringLiteral string_literal_parser; ParserCodec codec_parser; @@ -132,6 +136,24 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (!name_parser.parse(pos, name, expected)) return false; + const auto column_declaration = std::make_shared(); + tryGetIdentifierNameInto(name, column_declaration->name); + + /// This keyword may occur only in MODIFY COLUMN query. We check it here + /// because ParserDataType parses types as an arbitrary identifiers and + /// doesn't check that parsed string is existing data type. In this way + /// REMOVE keyword can be parsed as data type and further parsing will fail. + /// So we just check this keyword and in case of success return column + /// column declaration with name only. + if (s_remove.checkWithoutMoving(pos, expected)) + { + if (!check_keywords_after_name) + return false; + + node = column_declaration; + return true; + } + /** column name should be followed by type name if it * is not immediately followed by {DEFAULT, MATERIALIZED, ALIAS, COMMENT} */ @@ -197,9 +219,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E return false; } - const auto column_declaration = std::make_shared(); node = column_declaration; - tryGetIdentifierNameInto(name, column_declaration->name); if (type) { diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index caf98e911ab..3b062c2d689 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -43,6 +43,30 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +namespace +{ + +AlterCommand::RemoveProperty removePropertyFromString(const String & property) +{ + if (property.empty()) + return AlterCommand::RemoveProperty::NO_PROPERTY; + else if (property == "DEFAULT") + return AlterCommand::RemoveProperty::DEFAULT; + else if (property == "MATERIALIZED") + return AlterCommand::RemoveProperty::MATERIALIZED; + else if (property == "ALIAS") + return AlterCommand::RemoveProperty::ALIAS; + else if (property == "COMMENT") + return AlterCommand::RemoveProperty::COMMENT; + else if (property == "CODEC") + return AlterCommand::RemoveProperty::CODEC; + else if (property == "TTL") + return AlterCommand::RemoveProperty::TTL; + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot remove unknown property '{}'", property); +} + +} std::optional AlterCommand::parse(const ASTAlterCommand * command_ast) { @@ -111,8 +135,9 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.type = AlterCommand::MODIFY_COLUMN; const auto & ast_col_decl = command_ast->col_decl->as(); - command.column_name = ast_col_decl.name; + command.to_remove = removePropertyFromString(command_ast->remove_property); + if (ast_col_decl.type) { command.data_type = data_type_factory.get(ast_col_decl.type); @@ -237,6 +262,13 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.ttl = command_ast->ttl; return command; } + else if (command_ast->type == ASTAlterCommand::REMOVE_TTL) + { + AlterCommand command; + command.ast = command_ast->clone(); + command.type = AlterCommand::REMOVE_TTL; + return command; + } else if (command_ast->type == ASTAlterCommand::MODIFY_SETTING) { AlterCommand command; @@ -301,24 +333,45 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con { metadata.columns.modify(column_name, after_column, first, [&](ColumnDescription & column) { - if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false); - - if (comment) - column.comment = *comment; - - if (ttl) - column.ttl = ttl; - - if (data_type) - column.type = data_type; - - /// User specified default expression or changed - /// datatype. We have to replace default. - if (default_expression || data_type) + if (to_remove == RemoveProperty::DEFAULT + || to_remove == RemoveProperty::MATERIALIZED + || to_remove == RemoveProperty::ALIAS) { - column.default_desc.kind = default_kind; - column.default_desc.expression = default_expression; + column.default_desc = ColumnDefault{}; + } + else if (to_remove == RemoveProperty::CODEC) + { + column.codec.reset(); + } + else if (to_remove == RemoveProperty::COMMENT) + { + column.comment = String{}; + } + else if (to_remove == RemoveProperty::TTL) + { + column.ttl.reset(); + } + else + { + if (codec) + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false); + + if (comment) + column.comment = *comment; + + if (ttl) + column.ttl = ttl; + + if (data_type) + column.type = data_type; + + /// User specified default expression or changed + /// datatype. We have to replace default. + if (default_expression || data_type) + { + column.default_desc.kind = default_kind; + column.default_desc.expression = default_expression; + } } }); @@ -450,6 +503,10 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, const Context & con { metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key); } + else if (type == REMOVE_TTL) + { + metadata.table_ttl = TTLTableDescription{}; + } else if (type == MODIFY_QUERY) { metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, context); @@ -584,6 +641,10 @@ bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metada if (ignore) return false; + /// We remove properties on metadata level + if (isRemovingProperty() || type == REMOVE_TTL) + return false; + if (type == DROP_COLUMN || type == DROP_INDEX || type == RENAME_COLUMN) return true; @@ -636,6 +697,11 @@ bool AlterCommand::isTTLAlter(const StorageInMemoryMetadata & metadata) const return ttl_changed; } +bool AlterCommand::isRemovingProperty() const +{ + return to_remove != RemoveProperty::NO_PROPERTY; +} + std::optional AlterCommand::tryConvertToMutationCommand(StorageInMemoryMetadata & metadata, const Context & context) const { if (!isRequireMutationStage(metadata)) @@ -716,6 +782,8 @@ String alterTypeToString(const AlterCommand::Type type) return "MODIFY QUERY"; case AlterCommand::Type::RENAME_COLUMN: return "RENAME COLUMN"; + case AlterCommand::Type::REMOVE_TTL: + return "REMOVE TTL"; } __builtin_unreachable(); } @@ -783,14 +851,15 @@ void AlterCommands::prepare(const StorageInMemoryMetadata & metadata) if (!has_column && command.if_exists) command.ignore = true; - if (has_column && command.data_type) + if (has_column) { auto column_from_table = columns.get(command.column_name); - if (!command.default_expression && column_from_table.default_desc.expression) + if (command.data_type && !command.default_expression && column_from_table.default_desc.expression) { command.default_kind = column_from_table.default_desc.kind; command.default_expression = column_from_table.default_desc.expression; } + } } else if (command.type == AlterCommand::ADD_COLUMN) @@ -857,6 +926,70 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con if (command.codec) CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context.getSettingsRef().allow_suspicious_codecs); + auto column_default = all_columns.getDefault(column_name); + if (column_default) + { + if (command.to_remove == AlterCommand::RemoveProperty::DEFAULT && column_default->kind != ColumnDefaultKind::Default) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot remove DEFAULT from column {}, because column default type is {}. Use REMOVE {} to delete it", + backQuote(column_name), toString(column_default->kind), toString(column_default->kind)); + } + if (command.to_remove == AlterCommand::RemoveProperty::MATERIALIZED && column_default->kind != ColumnDefaultKind::Materialized) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot remove MATERIALIZED from column {}, because column default type is {}. Use REMOVE {} to delete it", + backQuote(column_name), toString(column_default->kind), toString(column_default->kind)); + } + if (command.to_remove == AlterCommand::RemoveProperty::ALIAS && column_default->kind != ColumnDefaultKind::Alias) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Cannot remove ALIAS from column {}, because column default type is {}. Use REMOVE {} to delete it", + backQuote(column_name), toString(column_default->kind), toString(column_default->kind)); + } + } + + if (command.isRemovingProperty()) + { + if (!column_default && command.to_remove == AlterCommand::RemoveProperty::DEFAULT) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have DEFAULT, cannot remove it", + backQuote(column_name)); + + if (!column_default && command.to_remove == AlterCommand::RemoveProperty::ALIAS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have ALIAS, cannot remove it", + backQuote(column_name)); + + if (!column_default && command.to_remove == AlterCommand::RemoveProperty::MATERIALIZED) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have MATERIALIZED, cannot remove it", + backQuote(column_name)); + + auto column_from_table = all_columns.get(column_name); + if (command.to_remove == AlterCommand::RemoveProperty::TTL && column_from_table.ttl == nullptr) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have TTL, cannot remove it", + backQuote(column_name)); + if (command.to_remove == AlterCommand::RemoveProperty::CODEC && column_from_table.codec == nullptr) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have TTL, cannot remove it", + backQuote(column_name)); + if (command.to_remove == AlterCommand::RemoveProperty::COMMENT && column_from_table.comment.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Column {} doesn't have COMMENT, cannot remove it", + backQuote(column_name)); + + } modified_columns.emplace(column_name); } @@ -966,6 +1099,10 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con throw Exception{"Cannot rename column from nested struct to normal column and vice versa", ErrorCodes::BAD_ARGUMENTS}; } } + else if (command.type == AlterCommand::REMOVE_TTL && !metadata.hasAnyTableTTL()) + { + throw Exception{"Table doesn't have any table TTL expression, cannot remove", ErrorCodes::BAD_ARGUMENTS}; + } /// Collect default expressions for MODIFY and ADD comands if (command.type == AlterCommand::MODIFY_COLUMN || command.type == AlterCommand::ADD_COLUMN) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index 3578507a361..c973b0b6a6f 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -37,6 +37,22 @@ struct AlterCommand MODIFY_SETTING, MODIFY_QUERY, RENAME_COLUMN, + REMOVE_TTL, + }; + + /// Which property user wants to remove from column + enum class RemoveProperty + { + NO_PROPERTY, + /// Default specifiers + DEFAULT, + MATERIALIZED, + ALIAS, + + /// Other properties + COMMENT, + CODEC, + TTL }; Type type; @@ -107,16 +123,13 @@ struct AlterCommand /// Target column name String rename_to; + /// What to remove from column (or TTL) + RemoveProperty to_remove = RemoveProperty::NO_PROPERTY; + static std::optional parse(const ASTAlterCommand * command); void apply(StorageInMemoryMetadata & metadata, const Context & context) const; - /// Checks that alter query changes data. For MergeTree: - /// * column files (data and marks) - /// * each part meta (columns.txt) - /// in each part on disk (it's not lightweight alter). - bool isModifyingData(const StorageInMemoryMetadata & metadata) const; - /// Check that alter command require data modification (mutation) to be /// executed. For example, cast from Date to UInt16 type can be executed /// without any data modifications. But column drop or modify from UInt16 to @@ -132,6 +145,9 @@ struct AlterCommand /// Checks that any TTL changed by alter bool isTTLAlter(const StorageInMemoryMetadata & metadata) const; + /// Command removing some property from column or table + bool isRemovingProperty() const; + /// If possible, convert alter command to mutation command. In other case /// return empty optional. Some storages may execute mutations after /// metadata changes. @@ -164,9 +180,6 @@ public: /// Commands have to be prepared before apply. void apply(StorageInMemoryMetadata & metadata, const Context & context) const; - /// At least one command modify data on disk. - bool isModifyingData(const StorageInMemoryMetadata & metadata) const; - /// At least one command modify settings. bool isSettingsAlter() const; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6258ffd40fe..65c0c5ac313 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -900,10 +900,17 @@ ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & m if (metadata_diff.ttl_table_changed) { - ParserTTLExpressionList parser; - auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - ttl_for_table_ast, new_metadata.columns, global_context, new_metadata.primary_key); + if (!metadata_diff.new_ttl_table.empty()) + { + ParserTTLExpressionList parser; + auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + ttl_for_table_ast, new_metadata.columns, global_context, new_metadata.primary_key); + } + else /// TTL was removed + { + new_metadata.table_ttl = TTLTableDescription{}; + } } } @@ -3818,7 +3825,12 @@ void StorageReplicatedMergeTree::alter( future_metadata_in_zk.partition_key = serializeAST(*future_metadata.partition_key.expression_list_ast); if (ast_to_str(future_metadata.table_ttl.definition_ast) != ast_to_str(current_metadata->table_ttl.definition_ast)) - future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.table_ttl.definition_ast); + { + if (future_metadata.table_ttl.definition_ast) + future_metadata_in_zk.ttl_table = serializeAST(*future_metadata.table_ttl.definition_ast); + else /// TTL was removed + future_metadata_in_zk.ttl_table = ""; + } String new_indices_str = future_metadata.secondary_indices.toString(); if (new_indices_str != current_metadata->secondary_indices.toString()) diff --git a/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.reference b/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.reference new file mode 100644 index 00000000000..e7e7c2e6ad1 --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.reference @@ -0,0 +1,4 @@ +CREATE TABLE default.no_prop_table\n(\n `some_column` UInt64\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.no_prop_table\n(\n `some_column` UInt64\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_no_prop_table\n(\n `some_column` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/01493_r_no_prop_table\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_no_prop_table\n(\n `some_column` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/01493_r_no_prop_table\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.sql b/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.sql new file mode 100644 index 00000000000..aceb4e8140d --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_no_property_zookeeper.sql @@ -0,0 +1,51 @@ +DROP TABLE IF EXISTS no_prop_table; + +CREATE TABLE no_prop_table +( + some_column UInt64 +) +ENGINE MergeTree() +ORDER BY tuple(); + +SHOW CREATE TABLE no_prop_table; + +-- just nothing happened +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE DEFAULT; --{serverError 36} +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE MATERIALIZED; --{serverError 36} +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE ALIAS; --{serverError 36} +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE CODEC; --{serverError 36} +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE COMMENT; --{serverError 36} +ALTER TABLE no_prop_table MODIFY COLUMN some_column REMOVE TTL; --{serverError 36} + +ALTER TABLE no_prop_table REMOVE TTL; --{serverError 36} + +SHOW CREATE TABLE no_prop_table; + +DROP TABLE IF EXISTS no_prop_table; + +DROP TABLE IF EXISTS r_no_prop_table; + +CREATE TABLE r_no_prop_table +( + some_column UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/test/01493_r_no_prop_table', '1') +ORDER BY tuple(); + +SHOW CREATE TABLE r_no_prop_table; + +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE DEFAULT; --{serverError 36} +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE MATERIALIZED; --{serverError 36} +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE ALIAS; --{serverError 36} +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE CODEC; --{serverError 36} +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE COMMENT; --{serverError 36} +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE TTL; --{serverError 36} + +ALTER TABLE r_no_prop_table REMOVE TTL; --{serverError 36} + +SHOW CREATE TABLE r_no_prop_table; + +ALTER TABLE r_no_prop_table MODIFY COLUMN some_column REMOVE ttl; --{serverError 36} +ALTER TABLE r_no_prop_table remove TTL; --{serverError 36} + +DROP TABLE IF EXISTS r_no_prop_table; diff --git a/tests/queries/0_stateless/01493_alter_remove_properties.reference b/tests/queries/0_stateless/01493_alter_remove_properties.reference new file mode 100644 index 00000000000..57b0ff22a4b --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_properties.reference @@ -0,0 +1,20 @@ +CREATE TABLE default.prop_table\n(\n `column_default` UInt64 DEFAULT 42,\n `column_materialized` UInt64 MATERIALIZED column_default * 42,\n `column_alias` UInt64 ALIAS column_default + 1,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date COMMENT \'Some comment\',\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +42 1764 43 str 2019-10-01 1 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64 DEFAULT 42,\n `column_materialized` UInt64 MATERIALIZED column_default * 42,\n `column_alias` UInt64 ALIAS column_default + 1,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64 DEFAULT 42,\n `column_materialized` UInt64 MATERIALIZED column_default * 42,\n `column_alias` UInt64 ALIAS column_default + 1,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +42 1764 0 str 2019-10-01 1 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64 DEFAULT 42,\n `column_materialized` UInt64 MATERIALIZED column_default * 42,\n `column_alias` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +42 1764 0 str 2019-10-01 1 +42 1764 33 trs 2020-01-01 2 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64 DEFAULT 42,\n `column_materialized` UInt64,\n `column_alias` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +42 1764 0 str 2019-10-01 1 +42 1764 33 trs 2020-01-01 2 +42 11 44 rts 2020-02-01 3 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64,\n `column_materialized` UInt64,\n `column_alias` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +42 1764 0 str 2019-10-01 1 +42 1764 33 trs 2020-01-01 2 +42 11 44 rts 2020-02-01 3 +0 22 55 tsr 2020-03-01 4 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64,\n `column_materialized` UInt64,\n `column_alias` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.prop_table\n(\n `column_default` UInt64,\n `column_materialized` UInt64,\n `column_alias` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +4 diff --git a/tests/queries/0_stateless/01493_alter_remove_properties.sql b/tests/queries/0_stateless/01493_alter_remove_properties.sql new file mode 100644 index 00000000000..01213ccd826 --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_properties.sql @@ -0,0 +1,72 @@ +DROP TABLE IF EXISTS prop_table; + +CREATE TABLE prop_table +( + column_default UInt64 DEFAULT 42, + column_materialized UInt64 MATERIALIZED column_default * 42, + column_alias UInt64 ALIAS column_default + 1, + column_codec String CODEC(ZSTD(10)), + column_comment Date COMMENT 'Some comment', + column_ttl UInt64 TTL column_comment + INTERVAL 1 MONTH +) +ENGINE MergeTree() +ORDER BY tuple() +TTL column_comment + INTERVAL 2 MONTH; + +SHOW CREATE TABLE prop_table; + +SYSTEM STOP TTL MERGES prop_table; + +INSERT INTO prop_table (column_codec, column_comment, column_ttl) VALUES ('str', toDate('2019-10-01'), 1); + +SELECT column_default, column_materialized, column_alias, column_codec, column_comment, column_ttl FROM prop_table; + +ALTER TABLE prop_table MODIFY COLUMN column_comment REMOVE COMMENT; + +SHOW CREATE TABLE prop_table; + +ALTER TABLE prop_table MODIFY COLUMN column_codec REMOVE CODEC; + +SHOW CREATE TABLE prop_table; + +ALTER TABLE prop_table MODIFY COLUMN column_alias REMOVE ALIAS; + +SELECT column_default, column_materialized, column_alias, column_codec, column_comment, column_ttl FROM prop_table; + +SHOW CREATE TABLE prop_table; + +INSERT INTO prop_table (column_alias, column_codec, column_comment, column_ttl) VALUES (33, 'trs', toDate('2020-01-01'), 2); + +SELECT column_default, column_materialized, column_alias, column_codec, column_comment, column_ttl FROM prop_table ORDER BY column_ttl; + +ALTER TABLE prop_table MODIFY COLUMN column_materialized REMOVE MATERIALIZED; + +SHOW CREATE TABLE prop_table; + +INSERT INTO prop_table (column_materialized, column_alias, column_codec, column_comment, column_ttl) VALUES (11, 44, 'rts', toDate('2020-02-01'), 3); + +SELECT column_default, column_materialized, column_alias, column_codec, column_comment, column_ttl FROM prop_table ORDER BY column_ttl; + +ALTER TABLE prop_table MODIFY COLUMN column_default REMOVE DEFAULT; + +SHOW CREATE TABLE prop_table; + +INSERT INTO prop_table (column_materialized, column_alias, column_codec, column_comment, column_ttl) VALUES (22, 55, 'tsr', toDate('2020-03-01'), 4); + +SELECT column_default, column_materialized, column_alias, column_codec, column_comment, column_ttl FROM prop_table ORDER BY column_ttl; + +ALTER TABLE prop_table REMOVE TTL; + +SHOW CREATE TABLE prop_table; + +ALTER TABLE prop_table MODIFY COLUMN column_ttl REMOVE TTL; + +SHOW CREATE TABLE prop_table; + +SYSTEM START TTL MERGES prop_table; + +OPTIMIZE TABLE prop_table FINAL; + +SELECT COUNT() FROM prop_table; + +DROP TABLE IF EXISTS prop_table; diff --git a/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.reference b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.reference new file mode 100644 index 00000000000..7cd2c81b460 --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.reference @@ -0,0 +1,21 @@ +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date COMMENT \'Some comment\',\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date COMMENT \'Some comment\',\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +====== remove column comment ====== +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String CODEC(ZSTD(10)),\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +====== remove column codec ====== +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64 DEFAULT 42,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +====== remove column default ====== +42 str 1 +0 tsr 2 +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64 TTL column_comment + toIntervalMonth(1)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +====== remove column TTL ====== +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nTTL column_comment + toIntervalMonth(2)\nSETTINGS index_granularity = 8192 +====== remove table TTL ====== +CREATE TABLE default.r_prop_table1\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.r_prop_table2\n(\n `column_default` UInt64,\n `column_codec` String,\n `column_comment` Date,\n `column_ttl` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test_01493/r_prop_table\', \'2\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +3 +3 diff --git a/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql new file mode 100644 index 00000000000..c095c4216fd --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql @@ -0,0 +1,92 @@ +DROP TABLE IF EXISTS r_prop_table1; +DROP TABLE IF EXISTS r_prop_table2; + +SET replication_alter_partitions_sync = 2; + +CREATE TABLE r_prop_table1 +( + column_default UInt64 DEFAULT 42, + column_codec String CODEC(ZSTD(10)), + column_comment Date COMMENT 'Some comment', + column_ttl UInt64 TTL column_comment + INTERVAL 1 MONTH +) +ENGINE ReplicatedMergeTree('/clickhouse/test_01493/r_prop_table', '1') +ORDER BY tuple() +TTL column_comment + INTERVAL 2 MONTH; + +CREATE TABLE r_prop_table2 +( + column_default UInt64 DEFAULT 42, + column_codec String CODEC(ZSTD(10)), + column_comment Date COMMENT 'Some comment', + column_ttl UInt64 TTL column_comment + INTERVAL 1 MONTH +) +ENGINE ReplicatedMergeTree('/clickhouse/test_01493/r_prop_table', '2') +ORDER BY tuple() +TTL column_comment + INTERVAL 2 MONTH; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +INSERT INTO r_prop_table1 (column_codec, column_comment, column_ttl) VALUES ('str', toDate('2020-10-01'), 1); + +SYSTEM SYNC REPLICA r_prop_table2; + +SELECT '====== remove column comment ======'; +ALTER TABLE r_prop_table1 MODIFY COLUMN column_comment REMOVE COMMENT; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +DETACH TABLE r_prop_table1; +ATTACH TABLE r_prop_table1; + +SELECT '====== remove column codec ======'; +ALTER TABLE r_prop_table2 MODIFY COLUMN column_codec REMOVE CODEC; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +SELECT '====== remove column default ======'; +ALTER TABLE r_prop_table2 MODIFY COLUMN column_default REMOVE DEFAULT; + +INSERT INTO r_prop_table1 (column_codec, column_comment, column_ttl) VALUES ('tsr', now(), 2); + +SYSTEM SYNC REPLICA r_prop_table2; + +SELECT column_default, column_codec, column_ttl FROM r_prop_table1 ORDER BY column_ttl; + +DETACH TABLE r_prop_table2; +ATTACH TABLE r_prop_table2; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +SELECT '====== remove column TTL ======'; +ALTER TABLE r_prop_table2 MODIFY COLUMN column_ttl REMOVE TTL; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +SELECT '====== remove table TTL ======'; +ALTER TABLE r_prop_table1 REMOVE TTL; + +INSERT INTO r_prop_table1 (column_codec, column_comment, column_ttl) VALUES ('rts', now() - INTERVAL 1 YEAR, 3); + +SYSTEM SYNC REPLICA r_prop_table2; + +DETACH TABLE r_prop_table2; +ATTACH TABLE r_prop_table2; + +SHOW CREATE TABLE r_prop_table1; +SHOW CREATE TABLE r_prop_table2; + +OPTIMIZE TABLE r_prop_table2 FINAL; + +SYSTEM SYNC REPLICA r_prop_table1; + +SELECT COUNT() FROM r_prop_table1; +SELECT COUNT() FROM r_prop_table2; + +DROP TABLE IF EXISTS r_prop_table1; +DROP TABLE IF EXISTS r_prop_table2; diff --git a/tests/queries/0_stateless/01493_alter_remove_wrong_default.reference b/tests/queries/0_stateless/01493_alter_remove_wrong_default.reference new file mode 100644 index 00000000000..27120d582e4 --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_wrong_default.reference @@ -0,0 +1 @@ +CREATE TABLE default.default_table\n(\n `key` UInt64 DEFAULT 42,\n `value1` UInt64 MATERIALIZED key * key,\n `value2` UInt64 ALIAS value1 * key\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01493_alter_remove_wrong_default.sql b/tests/queries/0_stateless/01493_alter_remove_wrong_default.sql new file mode 100644 index 00000000000..2099604ec13 --- /dev/null +++ b/tests/queries/0_stateless/01493_alter_remove_wrong_default.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS default_table; + +CREATE TABLE default_table ( + key UInt64 DEFAULT 42, + value1 UInt64 MATERIALIZED key * key, + value2 ALIAS value1 * key +) +ENGINE = MergeTree() +ORDER BY tuple(); + +ALTER TABLE default_table MODIFY COLUMN key REMOVE MATERIALIZED; --{serverError 36} +ALTER TABLE default_table MODIFY COLUMN key REMOVE ALIAS; --{serverError 36} + +ALTER TABLE default_table MODIFY COLUMN value1 REMOVE DEFAULT; --{serverError 36} +ALTER TABLE default_table MODIFY COLUMN value1 REMOVE ALIAS; --{serverError 36} + +ALTER TABLE default_table MODIFY COLUMN value2 REMOVE DEFAULT; --{serverError 36} +ALTER TABLE default_table MODIFY COLUMN value2 REMOVE MATERIALIZED; --{serverError 36} + +SHOW CREATE TABLE default_table; + +DROP TABLE IF EXISTS default_table; From a28867486792552775855d16404b1596c99e9d19 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 19 Sep 2020 17:20:49 +0300 Subject: [PATCH 209/390] Fix testflows tests. --- docker/test/testflows/runner/Dockerfile | 2 +- tests/testflows/ldap/tests/user_config.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 898552ade56..ed49743319c 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.48 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/tests/testflows/ldap/tests/user_config.py b/tests/testflows/ldap/tests/user_config.py index edc85a5877e..f609231b752 100644 --- a/tests/testflows/ldap/tests/user_config.py +++ b/tests/testflows/ldap/tests/user_config.py @@ -29,6 +29,7 @@ def empty_user_name(self, timeout=20): def empty_server_name(self, timeout=20): """Check that if server name is an empty string then login is not allowed. """ + message = "Exception: LDAP server name cannot be empty for user" servers = {"openldap1": { "host": "openldap1", "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" @@ -37,7 +38,8 @@ def empty_server_name(self, timeout=20): "errorcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] - login(servers, *users) + config = create_ldap_users_config_content(*users) + invalid_user_config(servers, config, message=message, tail=15, timeout=timeout) @TestScenario @Requirements( @@ -147,9 +149,6 @@ def ldap_and_password(self): with Then("I expect an error when I try to load the configuration file", description=error_message): invalid_user_config(servers, new_config, message=error_message, tail=16) - with And("I expect the authentication to fail when I try to login"): - login(servers, user, config=new_config) - @TestFeature @Name("user config") def feature(self, node="clickhouse1"): From a4337635ccb87cbcc5585672f2c9d5c9a8e218b0 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 19:42:11 +0300 Subject: [PATCH 210/390] Update single_fixed_string_groupby.xml --- tests/performance/single_fixed_string_groupby.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/single_fixed_string_groupby.xml b/tests/performance/single_fixed_string_groupby.xml index 10ca5f1ebcd..013de70c11a 100644 --- a/tests/performance/single_fixed_string_groupby.xml +++ b/tests/performance/single_fixed_string_groupby.xml @@ -10,8 +10,8 @@ INSERT INTO perf_lc_fixed_str_groupby SELECT ('number key ' || toString(number % 400)) AS a, ('number key ' || toString(number % 20)) AS b FROM numbers(30000000) - SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY a - SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY b + SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY a + SELECT count() FROM perf_lc_fixed_str_groupby GROUP BY b DROP TABLE IF EXISTS perf_lc_fixed_str_groupby From 5244653493f963eff2582726536ca6fd55093d14 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 19:45:27 +0300 Subject: [PATCH 211/390] Update maxmap.md --- docs/ru/sql-reference/aggregate-functions/reference/maxmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md b/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md index efac771666b..af817ee1d04 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/maxmap.md @@ -6,13 +6,13 @@ toc_priority: 143 Синтаксис: `maxMap(key, value)` or `maxMap(Tuple(key, value))` -Вычисляет максимальное значение массива `value` в соответствии с ключами, указанными в массиве `key`. +Вычисляет максимальные значения массива `value`, соответствующие ключам, указанным в массиве `key`. Передача кортежа ключей и массивов значений идентична передаче двух массивов ключей и значений. Количество элементов в параметрах `key` и `value` должно быть одинаковым для каждой суммируемой строки. -Возвращает кортеж из двух массивов: ключи в отсортированном порядке и значения, рассчитанные для соответствующих ключей. +Возвращает кортеж из двух массивов: ключи и значения, рассчитанные для соответствующих ключей. Пример: From 757be58fca4dd771231ee423d5f6ed1db8e90d5b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 19:45:43 +0300 Subject: [PATCH 212/390] Update maxmap.md --- docs/en/sql-reference/aggregate-functions/reference/maxmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md index ffb36f06e61..46f0881ba49 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md +++ b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md @@ -12,7 +12,7 @@ Passing a tuple of keys and value ​​arrays is identical to passing two array The number of elements in `key` and `value` must be the same for each row that is totaled. -Returns a tuple of two arrays: keys in sorted order, and values calculated for the corresponding keys. +Returns a tuple of two arrays: keys and values calculated for the corresponding keys. Example: From b2e9d0f20f9050213170451127262c36bccfddd2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 19:45:57 +0300 Subject: [PATCH 213/390] Update maxmap.md --- docs/en/sql-reference/aggregate-functions/reference/maxmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md index 46f0881ba49..c62502cf46e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md +++ b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md @@ -8,7 +8,7 @@ Syntax: `maxMap(key, value)` or `maxMap(Tuple(key, value))` Calculates the maximum from `value` array according to the keys specified in the `key` array. -Passing a tuple of keys and value ​​arrays is identical to passing two arrays of keys and values. +Passing a tuple of keys and value arrays is identical to passing two arrays of keys and values. The number of elements in `key` and `value` must be the same for each row that is totaled. From 92756f4635be4170969938e4be41fd20f1ec568f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 19:46:44 +0300 Subject: [PATCH 214/390] Update type-conversion-functions.md --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 7a57b94c4cd..987f9f6ae85 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -515,7 +515,7 @@ SELECT parseDateTimeBestEffort('10 20:19') ## parseDateTimeBestEffortUS {#parsedatetimebesteffortUS} -Эта функция похожа на [‘parseDateTimeBestEffort’](#parsedatetimebesteffort), но разница состоит в том, что в она использует американский формат даты (`MM/DD/YYYY` etc.) в случае многозначности. +Эта функция похожа на [‘parseDateTimeBestEffort’](#parsedatetimebesteffort), но разница состоит в том, что в она предполагает американский формат даты (`MM/DD/YYYY` etc.) в случае неоднозначности. **Синтаксис** From 25a1f8263113622f375caa45a1ab3beaf451e70d Mon Sep 17 00:00:00 2001 From: Mark Papadakis Date: Sun, 20 Sep 2020 20:26:44 +0300 Subject: [PATCH 215/390] Update ForkProcessor.cpp Got rid of an unnecessary local --- src/Processors/ForkProcessor.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Processors/ForkProcessor.cpp b/src/Processors/ForkProcessor.cpp index 7fa21c4236d..fe836ca740d 100644 --- a/src/Processors/ForkProcessor.cpp +++ b/src/Processors/ForkProcessor.cpp @@ -10,7 +10,6 @@ ForkProcessor::Status ForkProcessor::prepare() /// Check can output. - bool all_finished = true; bool all_can_push = true; size_t num_active_outputs = 0; @@ -18,7 +17,6 @@ ForkProcessor::Status ForkProcessor::prepare() { if (!output.isFinished()) { - all_finished = false; ++num_active_outputs; /// The order is important. @@ -27,8 +25,7 @@ ForkProcessor::Status ForkProcessor::prepare() } } - if (all_finished) - { + if (0 == num_active_outputs) { input.close(); return Status::Finished; } From ffe4f4b3668cd778216ef0ee098a773e71bb9edb Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 22:59:26 +0300 Subject: [PATCH 216/390] Update ForkProcessor.cpp --- src/Processors/ForkProcessor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Processors/ForkProcessor.cpp b/src/Processors/ForkProcessor.cpp index fe836ca740d..9b17f8ad5ca 100644 --- a/src/Processors/ForkProcessor.cpp +++ b/src/Processors/ForkProcessor.cpp @@ -25,7 +25,8 @@ ForkProcessor::Status ForkProcessor::prepare() } } - if (0 == num_active_outputs) { + if (0 == num_active_outputs) + { input.close(); return Status::Finished; } From 07f95d03994682fba72336695238a6d38c7b523a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 20 Sep 2020 23:07:17 +0300 Subject: [PATCH 217/390] Update lowcardinality.md --- docs/ru/sql-reference/data-types/lowcardinality.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/data-types/lowcardinality.md b/docs/ru/sql-reference/data-types/lowcardinality.md index fc10624600a..ec9e4e7588e 100644 --- a/docs/ru/sql-reference/data-types/lowcardinality.md +++ b/docs/ru/sql-reference/data-types/lowcardinality.md @@ -19,11 +19,11 @@ LowCardinality(data_type) ## Описание {#lowcardinality-dscr} -`LowCardinality` — это надстройка, изменяющая способ хранения и правила обработки данных. ClickHouse применяет [словарное кодирование](https://en.wikipedia.org/wiki/Dictionary_coder) в столбцы типа `LowCardinality`. Работа с данными, представленными в словарном виде, значительно увеличивает производительность запросов [SELECT](../statements/select/index.md) для многих приложений. +`LowCardinality` — это надстройка, изменяющая способ хранения и правила обработки данных. ClickHouse применяет [словарное кодирование](https://en.wikipedia.org/wiki/Dictionary_coder) в столбцы типа `LowCardinality`. Работа с данными, представленными в словарном виде, может значительно увеличивать производительность запросов [SELECT](../statements/select/index.md) для многих приложений. Эффективность использования типа данных `LowCarditality` зависит от разнообразия данных. Если словарь содержит менее 10 000 различных значений, ClickHouse в основном показывает более высокую эффективность чтения и хранения данных. Если же словарь содержит более 100 000 различных значений, ClickHouse может работать хуже, чем при использовании обычных типов данных. -При работе со строками используйте `LowCardinality` вместо [Enum](enum.md). `LowCardinality` обеспечивает большую гибкость в использовании и часто показывает такую же или более высокую эффективность. +При работе со строками, использование `LowCardinality` вместо [Enum](enum.md). `LowCardinality` обеспечивает большую гибкость в использовании и часто показывает такую же или более высокую эффективность. ## Пример @@ -56,4 +56,4 @@ ORDER BY id - [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). - [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). -- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). \ No newline at end of file +- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). From 5301de7656b4b0b92a8c28ecca119ce4e1935a75 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 20 Sep 2020 23:43:28 +0300 Subject: [PATCH 218/390] Control heavy translation units --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a879ad9b076..14f1fcb4a64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,11 @@ endif () include (cmake/find/ccache.cmake) +option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling" OFF) +if (ENABLE_CHECK_HEAVY_BUILDS) + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --rss=10000000 --cpu=600) +endif () + if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") set (CMAKE_BUILD_TYPE "RelWithDebInfo") message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = ${CMAKE_BUILD_TYPE}") From 30f96dc525a6ba3540322346df4a0b790aed5634 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 20 Sep 2020 23:41:36 +0300 Subject: [PATCH 219/390] Fix possible UAF from jobs in the ThreadPool on shutdown ThreadPoolImpl<>::worker signaling that job is done while still storing std::function<> object, and this can lead to problems on shutdown, since in in this cast std::function<> can refers to some global/static object that had been already destroyed (typical example is Logger). I believe that this is exactly what TSAN reports about (decoded manually, since llvm-symbolizer does not work in the test env):
``` - 2020-09-20 17:44:43 Write of size 8 at 0x7b1000008f78 by main thread (mutexes: write M1432): operator delete(void*, unsigned long) ??:0:0 Poco::Logger::~Logger() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:39:1 non-virtual thunk to Poco::Logger::~Logger() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:0:0 Poco::RefCountedObject::release() const /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/RefCountedObject.h:82:24 Poco::Logger::shutdown() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:346:16 Poco::AutoLoggerShutdown::~AutoLoggerShutdown() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:459:4 cxa_at_exit_wrapper(void*) ??:0:0 ``` ``` - 2020-09-20 17:44:43 Previous atomic read of size 4 at 0x7b1000008f78 by thread T116: __tsan_atomic32_load ??:0:0 int std::__1::__cxx_atomic_load(std::__1::__cxx_atomic_base_impl const*, std::__1::memory_order) /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:970:12 std::__1::__atomic_base::load(std::__1::memory_order) const /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:1487:17 std::__1::__atomic_base::operator int() const /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/atomic:1491:53 Poco::Logger::is(int) const /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/include/Poco/Logger.h:1264:9 MemoryTracker::logPeakMemoryUsage() const /build/obj-x86_64-linux-gnu/../src/Common/MemoryTracker.cpp:59:5 MemoryTracker::~MemoryTracker() /build/obj-x86_64-linux-gnu/../src/Common/MemoryTracker.cpp:46:13 DB::ThreadGroupStatus::~ThreadGroupStatus() /build/obj-x86_64-linux-gnu/../src/Common/ThreadStatus.h:51:7 std::__1::__shared_ptr_emplace >::__on_zero_shared() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3602:23 std::__1::__shared_count::__release_shared() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3440:9 std::__1::__shared_weak_count::__release_shared() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:3482:27 std::__1::shared_ptr::~shared_ptr() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:4207:19 DB::PipelineExecutor::executeImpl(unsigned long)::$_4::~$_4() /build/obj-x86_64-linux-gnu/../src/Processors/Executors/PipelineExecutor.cpp:720:34 ThreadFromGlobalPool::ThreadFromGlobalPool(DB::PipelineExecutor::executeImpl(unsigned long)::$_4&&)::'lambda'()::~() /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.h:161:54 std::__1::__compressed_pair_elem(DB::PipelineExecutor::executeImpl(unsigned long)::$_4&&)::'lambda'(), 0, false>::~__compressed_pair_elem() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2188:8 std::__1::__function::__alloc_func(DB::PipelineExecutor::executeImpl(unsigned long)::$_4&&)::'lambda'(), std::__1::allocator /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1559:38 std::__1::__function::__func(DB::PipelineExecutor::executeImpl(unsigned long)::$_4&&)::'lambda'(), std::__1::allocator /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1706:10 std::__1::__function::__value_func::~__value_func() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:1828:19 std::__1::function::~function() /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2460:43 ThreadPoolImpl::worker(std::__1::__list_iterator) /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:268:5 void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()::operator()() const /build/obj-x86_64-linux-gnu/../src/Common/ThreadPool.cpp:116:73 decltype(std::__1::forward(fp)(std::__1::forward::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'()>(fp0)...)) std::__1::__invoke::scheduleI> /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3519:1 void std::__1::__thread_execute >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional)::'lambda1'(> /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:273:5 void* std::__1::__thread_proxy >, void ThreadPoolImpl::scheduleImpl(std::__1::function, int, std::__1::optional /build/obj-x86_64-linux-gnu/../contrib/libcxx/include/thread:284:5 __tsan_thread_start_func ??:0:0 ``` ``` - 2020-09-20 17:44:43 Mutex M1432 (0x0000181213a8) created at: pthread_mutex_init ??:0:0 Poco::MutexImpl::MutexImpl() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Mutex_POSIX.cpp:64:6 Poco::Mutex::Mutex() /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Mutex.cpp:34:8 __cxx_global_var_init /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:28:15 _GLOBAL__sub_I_Logger.cpp /build/obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Logger.cpp:0:0 __libc_csu_init ??:0:0 ```
--- src/Common/ThreadPool.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 49516d777fb..f1fa82b7e68 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -233,6 +233,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::is_same_v ? CurrentMetrics::GlobalThreadActive : CurrentMetrics::LocalThreadActive); job(); + job = Job(); } catch (...) { From 035d7cb47f12bd8ffe7fa72af0d866e415752b83 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 20 Sep 2020 23:46:32 +0300 Subject: [PATCH 220/390] Enable control for heavy translation units --- docker/packager/binary/build.sh | 2 +- release | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index dc34e7297dc..fd70b03242b 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -17,7 +17,7 @@ ccache --show-stats ||: ccache --zero-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: rm -f CMakeCache.txt -cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS .. +cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER -DENABLE_CHECK_HEAVY_BUILDS=1 $CMAKE_FLAGS .. ninja $NINJA_FLAGS clickhouse-bundle mv ./programs/clickhouse* /output mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds diff --git a/release b/release index ec762b234fb..b20683a9caa 100755 --- a/release +++ b/release @@ -106,7 +106,7 @@ elif [[ $BUILD_TYPE == 'debug' ]]; then VERSION_POSTFIX+="+debug" fi -CMAKE_FLAGS=" $MALLOC_OPTS -DSANITIZE=$SANITIZER $CMAKE_FLAGS" +CMAKE_FLAGS=" $MALLOC_OPTS -DSANITIZE=$SANITIZER -DENABLE_CHECK_HEAVY_BUILDS=1 $CMAKE_FLAGS" [[ -n "$CMAKE_BUILD_TYPE" ]] && CMAKE_FLAGS=" -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE $CMAKE_FLAGS" export CMAKE_FLAGS From 6e1bd79d51958603be38907c622493bcad9b0c8e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Sep 2020 00:25:47 +0300 Subject: [PATCH 221/390] Add a test for #4476 --- .../0_stateless/01504_view_type_conversion.reference | 2 ++ .../0_stateless/01504_view_type_conversion.sql | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/01504_view_type_conversion.reference create mode 100644 tests/queries/0_stateless/01504_view_type_conversion.sql diff --git a/tests/queries/0_stateless/01504_view_type_conversion.reference b/tests/queries/0_stateless/01504_view_type_conversion.reference new file mode 100644 index 00000000000..e32c81f4c4a --- /dev/null +++ b/tests/queries/0_stateless/01504_view_type_conversion.reference @@ -0,0 +1,2 @@ +[0,1,2,3,4,5,6,7,8,9] +['0','1','2','3','4','5','6','7','8','9'] diff --git a/tests/queries/0_stateless/01504_view_type_conversion.sql b/tests/queries/0_stateless/01504_view_type_conversion.sql new file mode 100644 index 00000000000..0133ecaf409 --- /dev/null +++ b/tests/queries/0_stateless/01504_view_type_conversion.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS testv; + +create view testv(a UInt32) as select number a from numbers(10); +select groupArray(a) from testv; + +DROP TABLE testv; + +create view testv(a String) as select number a from numbers(10); +select groupArray(a) from testv; + +DROP TABLE testv; From fe0f8117afb5aa6c24538a202816c36b3072459a Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sun, 20 Sep 2020 19:22:57 -0300 Subject: [PATCH 222/390] more tests for CTE --- ...495_subqueries_in_with_statement.reference | 18 ++++++ .../01495_subqueries_in_with_statement.sql | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference b/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference index 8e851cd3ba1..16ca3450a74 100644 --- a/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference +++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement.reference @@ -12,3 +12,21 @@ 4 5 4 5 4 5 +--------------------------- +42 +42 +0 +42 +42 +42 +\N +42 +42 +42 +42 +42 +45 +\N +\N +42 +42 diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql index 9ec921a9d4c..f34f4aaa6e0 100644 --- a/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql +++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql @@ -11,3 +11,64 @@ SELECT * FROM (WITH test1 AS (SELECT toInt32(*) i FROM numbers(5)) SELECT * FROM WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT toInt64(4) i, toInt64(5) j FROM numbers(3) WHERE (i, j) IN test1; DROP TABLE IF EXISTS test1; + +select '---------------------------'; + +set empty_result_for_aggregation_by_empty_set = 0; + +WITH test1 AS (SELECT number-1 as n FROM numbers(42)) +SELECT max(n+1)+1 z FROM test1; + +WITH test1 AS (SELECT number-1 as n FROM numbers(42)) +SELECT max(n+1)+1 z FROM test1 join test1 x using n having z - 1 = (select min(n-1)+41 from test1) + 2; + +WITH test1 AS (SELECT number-1 as n FROM numbers(4442) limit 100) +SELECT max(n) FROM test1 where n=422; + +WITH test1 AS (SELECT number-1 as n FROM numbers(4442) limit 100) +SELECT max(n) FROM test1 where n=42; + +drop table if exists with_test ; +create table with_test engine=Memory as select cast(number-1 as Nullable(Int64)) n from numbers(10000); + +WITH test1 AS (SELECT n FROM with_test where n <= 40) +SELECT max(n+1)+1 z FROM test1 join test1 x using (n) having max(n+1)+1 - 1 = (select min(n-1)+41 from test1) + 2; + +WITH test1 AS (SELECT n FROM with_test where n <= 40) +SELECT max(n+1)+1 z FROM test1 join test1 x using (n) having z - 1 = (select min(n-1)+41 from test1) + 2; + +WITH test1 AS (SELECT n FROM with_test limit 100) +SELECT max(n) FROM test1 where n=422; + +WITH test1 AS (SELECT n FROM with_test limit 100) +SELECT max(n) FROM test1 where n=42; + +WITH test1 AS (SELECT n FROM with_test where n = 42 limit 100) +SELECT max(n) FROM test1 where n=42; + +WITH test1 AS (SELECT n FROM with_test where n = 42 or 1=1 limit 100) +SELECT max(n) FROM test1 where n=42; + +WITH test1 AS (SELECT n, null as b FROM with_test where n = 42 or b is null limit 100) +SELECT max(n) FROM test1 where n=42; + +WITH test1 AS (SELECT n, null b FROM with_test where b is null) +SELECT max(n) FROM test1 where n=42; + +WITH test1 AS (SELECT n, null b FROM with_test where b is null or 1=1) +SELECT max(n) FROM test1 where n=45; + +WITH test1 AS (SELECT n, null b FROM with_test where b is null and n = 42) +SELECT max(n) FROM test1 where n=45; + +WITH test1 AS (SELECT n, null b FROM with_test where 1=1 and n = 42) +SELECT max(n) FROM test1 where n=45; + +WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where 1=0 or n = 42 limit 4) +SELECT max(n) m FROM test1 where test1.m=43 having max(n)=42; + +WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where n = 42 limit 4) +SELECT max(n) m FROM test1 where b is null and test1.m=43 having m=42 limit 4; + +drop table with_test ; + From aa25df9d4538f87f56430f34da986e3f6e03cb88 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sun, 20 Sep 2020 19:43:19 -0300 Subject: [PATCH 223/390] order by added for quieries with limit --- .../01495_subqueries_in_with_statement.sql | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql index f34f4aaa6e0..8102ed29fa8 100644 --- a/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql +++ b/tests/queries/0_stateless/01495_subqueries_in_with_statement.sql @@ -16,40 +16,40 @@ select '---------------------------'; set empty_result_for_aggregation_by_empty_set = 0; -WITH test1 AS (SELECT number-1 as n FROM numbers(42)) +WITH test1 AS (SELECT number-1 as n FROM numbers(42)) SELECT max(n+1)+1 z FROM test1; -WITH test1 AS (SELECT number-1 as n FROM numbers(42)) +WITH test1 AS (SELECT number-1 as n FROM numbers(42)) SELECT max(n+1)+1 z FROM test1 join test1 x using n having z - 1 = (select min(n-1)+41 from test1) + 2; -WITH test1 AS (SELECT number-1 as n FROM numbers(4442) limit 100) +WITH test1 AS (SELECT number-1 as n FROM numbers(4442) order by n limit 100) SELECT max(n) FROM test1 where n=422; -WITH test1 AS (SELECT number-1 as n FROM numbers(4442) limit 100) +WITH test1 AS (SELECT number-1 as n FROM numbers(4442) order by n limit 100) SELECT max(n) FROM test1 where n=42; drop table if exists with_test ; create table with_test engine=Memory as select cast(number-1 as Nullable(Int64)) n from numbers(10000); -WITH test1 AS (SELECT n FROM with_test where n <= 40) +WITH test1 AS (SELECT n FROM with_test where n <= 40) SELECT max(n+1)+1 z FROM test1 join test1 x using (n) having max(n+1)+1 - 1 = (select min(n-1)+41 from test1) + 2; -WITH test1 AS (SELECT n FROM with_test where n <= 40) +WITH test1 AS (SELECT n FROM with_test where n <= 40) SELECT max(n+1)+1 z FROM test1 join test1 x using (n) having z - 1 = (select min(n-1)+41 from test1) + 2; -WITH test1 AS (SELECT n FROM with_test limit 100) +WITH test1 AS (SELECT n FROM with_test order by n limit 100) SELECT max(n) FROM test1 where n=422; -WITH test1 AS (SELECT n FROM with_test limit 100) +WITH test1 AS (SELECT n FROM with_test order by n limit 100) SELECT max(n) FROM test1 where n=42; -WITH test1 AS (SELECT n FROM with_test where n = 42 limit 100) +WITH test1 AS (SELECT n FROM with_test where n = 42 order by n limit 100) SELECT max(n) FROM test1 where n=42; -WITH test1 AS (SELECT n FROM with_test where n = 42 or 1=1 limit 100) +WITH test1 AS (SELECT n FROM with_test where n = 42 or 1=1 order by n limit 100) SELECT max(n) FROM test1 where n=42; -WITH test1 AS (SELECT n, null as b FROM with_test where n = 42 or b is null limit 100) +WITH test1 AS (SELECT n, null as b FROM with_test where n = 42 or b is null order by n limit 100) SELECT max(n) FROM test1 where n=42; WITH test1 AS (SELECT n, null b FROM with_test where b is null) @@ -61,14 +61,13 @@ SELECT max(n) FROM test1 where n=45; WITH test1 AS (SELECT n, null b FROM with_test where b is null and n = 42) SELECT max(n) FROM test1 where n=45; -WITH test1 AS (SELECT n, null b FROM with_test where 1=1 and n = 42) +WITH test1 AS (SELECT n, null b FROM with_test where 1=1 and n = 42 order by n) SELECT max(n) FROM test1 where n=45; -WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where 1=0 or n = 42 limit 4) +WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where 1=0 or n = 42 order by n limit 4) SELECT max(n) m FROM test1 where test1.m=43 having max(n)=42; -WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where n = 42 limit 4) +WITH test1 AS (SELECT n, null b, n+1 m FROM with_test where n = 42 order by n limit 4) SELECT max(n) m FROM test1 where b is null and test1.m=43 having m=42 limit 4; drop table with_test ; - From 0119eb9c0dbafb26fe6acff06f03e1773e78f566 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2020 06:17:03 +0000 Subject: [PATCH 224/390] Bump mkdocs-macros-plugin from 0.4.9 to 0.4.13 in /docs/tools Bumps [mkdocs-macros-plugin](https://github.com/fralau/mkdocs_macros_plugin) from 0.4.9 to 0.4.13. - [Release notes](https://github.com/fralau/mkdocs_macros_plugin/releases) - [Commits](https://github.com/fralau/mkdocs_macros_plugin/commits) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index a3949892829..e31e43b99cd 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -18,7 +18,7 @@ Markdown==3.2.1 MarkupSafe==1.1.1 mkdocs==1.1.2 mkdocs-htmlproofer-plugin==0.0.3 -mkdocs-macros-plugin==0.4.9 +mkdocs-macros-plugin==0.4.13 nltk==3.5 nose==1.3.7 protobuf==3.13.0 From 27613f834681ef77b24e0e6f01e67a26017b61b0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Sep 2020 10:38:11 +0300 Subject: [PATCH 225/390] Fix symbolizers path in test images --- docker/test/base/Dockerfile | 5 ++--- docker/test/integration/base/Dockerfile | 9 +-------- docker/test/stateless/run.sh | 7 ------- docker/test/stateless_unbundled/run.sh | 7 ------- docker/test/stress/run.sh | 2 -- docker/test/unit/Dockerfile | 9 +-------- docker/test/unit/perfraw/default.profraw | 0 7 files changed, 4 insertions(+), 35 deletions(-) create mode 100644 docker/test/unit/perfraw/default.profraw diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 8117d2907bc..8adaf5ab543 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,7 @@ # docker build -t yandex/clickhouse-test-base . -FROM ubuntu:19.10 +FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 RUN apt-get update \ && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ @@ -43,7 +43,6 @@ RUN apt-get update \ llvm-${LLVM_VERSION} \ moreutils \ perl \ - perl \ pigz \ pkg-config \ tzdata \ diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 53627c78208..35decd907c0 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,5 +1,5 @@ # docker build -t yandex/clickhouse-integration-test . -FROM ubuntu:19.10 +FROM yandex/clickhouse-test-base RUN apt-get update \ && env DEBIAN_FRONTEND=noninteractive apt-get -y install \ @@ -8,7 +8,6 @@ RUN apt-get update \ libreadline-dev \ libicu-dev \ bsdutils \ - llvm-9 \ gdb \ unixodbc \ odbcinst \ @@ -29,9 +28,3 @@ RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-od ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Sanitizer options -RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ - echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \ - ln -s /usr/lib/llvm-9/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 4a9ad891883..b6b48cd0943 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -48,13 +48,6 @@ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml -echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment -echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment -echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment - service zookeeper start sleep 5 service clickhouse-server start && sleep 5 diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index 4a9ad891883..b6b48cd0943 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -48,13 +48,6 @@ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml -echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment -echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment -echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment -echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment - service zookeeper start sleep 5 service clickhouse-server start && sleep 5 diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 47c8603babb..8295e90b3ef 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -43,8 +43,6 @@ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/u ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -echo "TSAN_OPTIONS='halt_on_error=1 history_size=7 ignore_noninstrumented_modules=1 verbosity=1'" >> /etc/environment -echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment start diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index ae5ea1820b0..0f65649fb76 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -5,12 +5,5 @@ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get install gdb -CMD ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \ - echo "TSAN_OPTIONS='halt_on_error=1 history_size=7'" >> /etc/environment; \ - echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \ - echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \ - service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; \ +CMD service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; \ gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt diff --git a/docker/test/unit/perfraw/default.profraw b/docker/test/unit/perfraw/default.profraw new file mode 100644 index 00000000000..e69de29bb2d From fd469f1266c61a85b5d898255243a1550249c4b6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Sep 2020 10:41:23 +0300 Subject: [PATCH 226/390] Remove redundant file --- docker/test/unit/perfraw/default.profraw | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docker/test/unit/perfraw/default.profraw diff --git a/docker/test/unit/perfraw/default.profraw b/docker/test/unit/perfraw/default.profraw deleted file mode 100644 index e69de29bb2d..00000000000 From aa3905989018bde484e78d060a792b8571c2ded6 Mon Sep 17 00:00:00 2001 From: Artemeey Date: Mon, 21 Sep 2020 11:10:27 +0300 Subject: [PATCH 227/390] Update storage_policies.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Описание move_factor --- docs/ru/operations/system-tables/storage_policies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/storage_policies.md b/docs/ru/operations/system-tables/storage_policies.md index f937654ab9a..df5c920b5ba 100644 --- a/docs/ru/operations/system-tables/storage_policies.md +++ b/docs/ru/operations/system-tables/storage_policies.md @@ -9,7 +9,7 @@ - `volume_priority` ([UInt64](../../sql-reference/data-types/int-uint.md)) — порядковый номер тома согласно конфигурации. - `disks` ([Array(String)](../../sql-reference/data-types/array.md)) — имена дисков, содержащихся в политике хранения. - `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — максимальный размер куска данных, который может храниться на дисках тома (0 — без ограничений). -- `move_factor` ([Float64](../../sql-reference/data-types/float.md))\` — доля свободного места, при превышении которой данные начинают перемещаться на следующий том. +- `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1). Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице. From a7aa714a280b07f9074a33ee6b8301a094bb2f9e Mon Sep 17 00:00:00 2001 From: Artemeey Date: Mon, 21 Sep 2020 11:14:37 +0300 Subject: [PATCH 228/390] Update mergetree.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit описание move_factor --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 881c11152cd..2f89317a0eb 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -563,7 +563,7 @@ ALTER TABLE example_table - `volume_name_N` — название тома. Названия томов должны быть уникальны. - `disk` — диск, находящийся внутри тома. - `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома. -- `move_factor` — доля свободного места, при превышении которого данные начинают перемещаться на следующий том, если он есть (по умолчанию 0.1). +- `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1). Примеры конфигураций: From 54c1f04b3873726a48b3271c72f738ab384aa15b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 21 Sep 2020 11:36:12 +0300 Subject: [PATCH 229/390] Review fixes. --- src/Interpreters/InterpreterSelectQuery.cpp | 3 --- src/Processors/QueryPipeline.cpp | 2 +- src/Processors/QueryPipeline.h | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 33fa81383a3..5cdc9bfa5b5 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1899,9 +1899,6 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_p const Settings & settings = context->getSettingsRef(); - if (subqueries_for_sets.empty()) - return; - SizeLimits limits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); addCreatingSetsStep(query_plan, std::move(subqueries_for_sets), limits, *context); } diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index c8218023a57..4cbb4d9edb7 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -282,7 +282,7 @@ void QueryPipeline::addCreatingSetsTransform(const Block & res_header, SubqueryF pipe.addTransform(std::move(transform), totals_port, nullptr); } -void QueryPipeline::addDelayingPipeline(QueryPipeline pipeline) +void QueryPipeline::addPipelineBefore(QueryPipeline pipeline) { checkInitializedAndNotCompleted(); assertBlocksHaveEqualStructure(getHeader(), pipeline.getHeader(), "QueryPipeline"); diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 2a318a53b76..80ae1d591a4 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -89,7 +89,7 @@ public: /// Add other pipeline and execute it before current one. /// Pipeline must have same header. - void addDelayingPipeline(QueryPipeline pipeline); + void addPipelineBefore(QueryPipeline pipeline); void addCreatingSetsTransform(const Block & res_header, SubqueryForSet subquery_for_set, const SizeLimits & limits, const Context & context); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 780e8493b40..5868a7045f7 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -95,7 +95,7 @@ QueryPipelinePtr CreatingSetsStep::updatePipeline(QueryPipelines pipelines) delayed_pipeline = std::move(*pipelines.front()); QueryPipelineProcessorsCollector collector(*main_pipeline, this); - main_pipeline->addDelayingPipeline(std::move(delayed_pipeline)); + main_pipeline->addPipelineBefore(std::move(delayed_pipeline)); auto added_processors = collector.detachProcessors(); processors.insert(processors.end(), added_processors.begin(), added_processors.end()); From 1bd76c820c035a7802b8448d0dd495469500e673 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Sep 2020 12:05:10 +0300 Subject: [PATCH 230/390] Back to 19.10 --- docker/test/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 8adaf5ab543..aa3f1d738c2 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,5 +1,5 @@ # docker build -t yandex/clickhouse-test-base . -FROM ubuntu:20.04 +FROM ubuntu:19.10 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 From ef85ce90c345fb593878ff1bd0c3e0729c729065 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 21 Sep 2020 13:17:33 +0300 Subject: [PATCH 231/390] fix build --- src/Common/renameat2.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 5139a165c91..2eaae491aba 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -79,10 +79,9 @@ static bool renameat2(const std::string & old_path, const std::string & new_path #define RENAME_NOREPLACE -1 #define RENAME_EXCHANGE -1 -[[noreturn]] -static void renameat2(const std::string &, const std::string &, int) +static bool renameat2(const std::string &, const std::string &, int) { - throw Exception("Compiled without renameat2() support", ErrorCodes::UNSUPPORTED_METHOD); + return false } #endif From 450c1a5709c859285887ea7e370949b74b99f115 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 21 Sep 2020 13:19:02 +0300 Subject: [PATCH 232/390] Update renameat2.cpp --- src/Common/renameat2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 2eaae491aba..02acd8331d2 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -50,7 +50,7 @@ static bool supportsRenameat2Impl() static bool renameat2(const std::string & old_path, const std::string & new_path, int flags) { - if(!supportsRenameat2()) + if (!supportsRenameat2()) return false; if (old_path.empty() || new_path.empty()) throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR); From efa493fbcb532772e46d2f43cfe8aee8931a9539 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 21 Sep 2020 13:29:31 +0300 Subject: [PATCH 233/390] Update SourceVisitor for GatherUtils. --- .../GatherUtils/ArraySourceVisitor.h | 10 +-- src/Functions/GatherUtils/Selectors.h | 76 +++++-------------- src/Functions/GatherUtils/concat.cpp | 61 +++++++++------ src/Functions/GatherUtils/has_all.cpp | 48 +++++++++++- src/Functions/GatherUtils/has_any.cpp | 48 +++++++++++- src/Functions/GatherUtils/has_substr.cpp | 48 +++++++++++- .../GatherUtils/sliceDynamicOffsetBounded.cpp | 32 +++++++- .../sliceDynamicOffsetUnbounded.cpp | 31 +++++++- .../sliceFromLeftConstantOffsetBounded.cpp | 31 +++++++- .../sliceFromLeftConstantOffsetUnbounded.cpp | 31 +++++++- .../sliceFromRightConstantOffsetBounded.cpp | 31 +++++++- .../sliceFromRightConstantOffsetUnbounded.cpp | 31 +++++++- 12 files changed, 364 insertions(+), 114 deletions(-) diff --git a/src/Functions/GatherUtils/ArraySourceVisitor.h b/src/Functions/GatherUtils/ArraySourceVisitor.h index 989f9ae918a..0446839325e 100644 --- a/src/Functions/GatherUtils/ArraySourceVisitor.h +++ b/src/Functions/GatherUtils/ArraySourceVisitor.h @@ -19,12 +19,12 @@ struct ConstSource; using NumericArraySources = typename TypeListMap::Type; using BasicArraySources = typename AppendToTypeList::Type; -using NullableArraySources = typename TypeListMap::Type; -using BasicAndNullableArraySources = typename TypeListConcat::Type; -using ConstArraySources = typename TypeListMap::Type; -using TypeListArraySources = typename TypeListConcat::Type; +//using NullableArraySources = typename TypeListMap::Type; +//using BasicAndNullableArraySources = typename TypeListConcat::Type; +//using ConstArraySources = typename TypeListMap::Type; +//using TypeListArraySources = typename TypeListConcat::Type; -class ArraySourceVisitor : public ApplyTypeListForClass::Type +class ArraySourceVisitor : public ApplyTypeListForClass::Type { protected: ~ArraySourceVisitor() = default; diff --git a/src/Functions/GatherUtils/Selectors.h b/src/Functions/GatherUtils/Selectors.h index 9c96b36460d..bbe631a6a3a 100644 --- a/src/Functions/GatherUtils/Selectors.h +++ b/src/Functions/GatherUtils/Selectors.h @@ -32,20 +32,30 @@ void callSelectMemberFunctionWithTupleArgument(Tuple & tuple, Args && ... args) callSelectMemberFunctionWithTupleArgument(tuple, args ..., std::get(tuple)); } +template +void callSelectSource(bool is_const, bool is_nullable, Tuple & tuple, Args && ... args) +{ + if constexpr (index == std::tuple_size::value) + Base::selectSource(is_const, is_nullable, args ...); + else + callSelectSource(is_const, is_nullable, tuple, args ..., std::get(tuple)); +} + template struct ArraySourceSelectorVisitor final : public ArraySourceVisitorImpl> { - explicit ArraySourceSelectorVisitor(Args && ... args) : packed_args(args ...) {} + explicit ArraySourceSelectorVisitor(IArraySource & source, Args && ... args) : packed_args(args ...), array_source(source) {} using Tuple = std::tuple; template void visitImpl(Source & source) { - callSelectMemberFunctionWithTupleArgument(packed_args, source); + callSelectSource(array_source.isConst(), array_source.isNullable(), packed_args, source); } Tuple packed_args; + IArraySource & array_source; }; template @@ -54,7 +64,7 @@ struct ArraySourceSelector template static void select(IArraySource & source, Args && ... args) { - ArraySourceSelectorVisitor visitor(args ...); + ArraySourceSelectorVisitor visitor(source, args ...); source.accept(visitor); } }; @@ -87,56 +97,6 @@ struct ArraySinkSelector } }; - -template -struct ValueSourceSelectorVisitor final : public ValueSourceVisitorImpl> -{ - explicit ValueSourceSelectorVisitor(Args && ... args) : packed_args(args ...) {} - - using Tuple = std::tuple; - - template - void visitImpl(Source & source) - { - callSelectMemberFunctionWithTupleArgument(packed_args, source); - } - - Tuple packed_args; -}; - -template -struct ValueSourceSelector -{ - template - static void select(IValueSource & source, Args && ... args) - { - ValueSourceSelectorVisitor visitor(args ...); - source.accept(visitor); - } -}; - -template -struct ArraySinkSourceSelector -{ - template - static void select(IArraySource & source, IArraySink & sink, Args && ... args) - { - ArraySinkSelector::select(sink, source, args ...); - } - - template - static void selectImpl(Sink && sink, IArraySource & source, Args && ... args) - { - ArraySourceSelector::select(source, sink, args ...); - } - - template - static void selectImpl(Source && source, Sink && sink, Args && ... args) - { - Base::selectSourceSink(source, sink, args ...); - } -}; - template struct ArraySourcePairSelector { @@ -147,15 +107,17 @@ struct ArraySourcePairSelector } template - static void selectImpl(FirstSource && first, IArraySource & second, Args && ... args) + static void selectSource(bool is_const, bool is_nullable, FirstSource && first, IArraySource & second, Args && ... args) { - ArraySourceSelector::select(second, first, args ...); + ArraySourceSelector::select(second, is_const, is_nullable, first, args ...); } template - static void selectImpl(SecondSource && second, FirstSource && first, Args && ... args) + static void selectSource(bool is_second_const, bool is_second_nullable, SecondSource && second, + bool is_first_const, bool is_first_nullable, FirstSource && first, Args && ... args) { - Base::selectSourcePair(first, second, args ...); + Base::selectSourcePair(is_first_const, is_first_nullable, first, + is_second_const, is_second_nullable, second, args ...); } }; diff --git a/src/Functions/GatherUtils/concat.cpp b/src/Functions/GatherUtils/concat.cpp index 3435baf147e..4fe6948ab56 100644 --- a/src/Functions/GatherUtils/concat.cpp +++ b/src/Functions/GatherUtils/concat.cpp @@ -23,37 +23,54 @@ struct ArrayConcat : public ArraySourceSelector { using Sources = std::vector>; +// template +// static void selectSource(Source && source, const Sources & sources, ColumnArray::MutablePtr & result) +// { +// using SourceType = typename std::decay::type; +// using Sink = typename SourceType::SinkType; +// result = ColumnArray::create(source.createValuesColumn()); +// Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); +// +// concat(sources, std::move(sink)); +// } +// +// template +// static void selectImpl(ConstSource && source, const Sources & sources, ColumnArray::MutablePtr & result) +// { +// using SourceType = typename std::decay::type; +// using Sink = typename SourceType::SinkType; +// result = ColumnArray::create(source.createValuesColumn()); +// Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); +// +// concat(sources, std::move(sink)); +// } + template - static void selectImpl(Source && source, const Sources & sources, ColumnArray::MutablePtr & result) + static void selectSource(bool /*is_const*/, bool is_nullable, Source & source, const Sources & sources, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - concat(sources, std::move(sink)); - } + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; - template - static void selectImpl(ConstSource && source, const Sources & sources, ColumnArray::MutablePtr & result) - { - using SourceType = typename std::decay::type; - using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + auto & nullable_source = static_cast(source); - concat(sources, std::move(sink)); - } - template - static void selectImpl(ConstSource & source, const Sources & sources, ColumnArray::MutablePtr & result) - { - using SourceType = typename std::decay::type; - using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - concat(sources, std::move(sink)); + concat(sources, std::move(sink)); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + concat(sources, std::move(sink)); + } } }; diff --git a/src/Functions/GatherUtils/has_all.cpp b/src/Functions/GatherUtils/has_all.cpp index 491631d0c7c..6e34a851c02 100644 --- a/src/Functions/GatherUtils/has_all.cpp +++ b/src/Functions/GatherUtils/has_all.cpp @@ -11,9 +11,53 @@ namespace struct ArrayHasAllSelectArraySourcePair : public ArraySourcePairSelector { template - static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) { - arrayAllAny(first, second, result); + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } } }; diff --git a/src/Functions/GatherUtils/has_any.cpp b/src/Functions/GatherUtils/has_any.cpp index 6ad0eeb8dc0..b7a8c9f620d 100644 --- a/src/Functions/GatherUtils/has_any.cpp +++ b/src/Functions/GatherUtils/has_any.cpp @@ -11,9 +11,53 @@ namespace struct ArrayHasAnySelectArraySourcePair : public ArraySourcePairSelector { template - static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) { - arrayAllAny(first, second, result); + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } } }; diff --git a/src/Functions/GatherUtils/has_substr.cpp b/src/Functions/GatherUtils/has_substr.cpp index fe16c423428..244a1d21633 100644 --- a/src/Functions/GatherUtils/has_substr.cpp +++ b/src/Functions/GatherUtils/has_substr.cpp @@ -11,9 +11,53 @@ namespace struct ArrayHasSubstrSelectArraySourcePair : public ArraySourcePairSelector { template - static void selectSourcePair(FirstSource && first, SecondSource && second, ColumnUInt8 & result) + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) { - arrayAllAny(first, second, result); + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } } }; diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp index 5222bf525cc..f8f46a2ac49 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetBounded.cpp @@ -13,13 +13,37 @@ namespace struct SliceDynamicOffsetBoundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, const IColumn & offset_column, const IColumn & length_column, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, + const IColumn & offset_column, const IColumn & length_column, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceDynamicOffsetBounded(source, sink, offset_column, length_column); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceDynamicOffsetBounded(static_cast &>(source), sink, offset_column, length_column); + else + sliceDynamicOffsetBounded(static_cast(source), sink, offset_column, length_column); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceDynamicOffsetBounded(static_cast &>(source), sink, offset_column, length_column); + else + sliceDynamicOffsetBounded(source, sink, offset_column, length_column); + } } }; diff --git a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp index fa98028f36f..2aa6a8903a4 100644 --- a/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceDynamicOffsetUnbounded.cpp @@ -14,13 +14,36 @@ struct SliceDynamicOffsetUnboundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, const IColumn & offset_column, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, const IColumn & offset_column, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceDynamicOffsetUnbounded(source, sink, offset_column); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); + else + sliceDynamicOffsetUnbounded(static_cast(source), sink, offset_column); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceDynamicOffsetUnbounded(static_cast &>(source), sink, offset_column); + else + sliceDynamicOffsetUnbounded(source, sink, offset_column); + } } }; diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp index 96ea1e076e1..404cfa1b10d 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetBounded.cpp @@ -14,13 +14,36 @@ struct SliceFromLeftConstantOffsetBoundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceFromLeftConstantOffsetBounded(source, sink, offset, length); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else + sliceFromLeftConstantOffsetBounded(static_cast(source), sink, offset, length); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromLeftConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else + sliceFromLeftConstantOffsetBounded(source, sink, offset, length); + } } }; diff --git a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp index a6b5f799c80..1a7fb03a275 100644 --- a/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromLeftConstantOffsetUnbounded.cpp @@ -14,13 +14,36 @@ struct SliceFromLeftConstantOffsetUnboundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, size_t & offset, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceFromLeftConstantOffsetUnbounded(source, sink, offset); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else + sliceFromLeftConstantOffsetUnbounded(static_cast(source), sink, offset); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromLeftConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else + sliceFromLeftConstantOffsetUnbounded(source, sink, offset); + } } }; diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp index 35833950cfe..faa6c6fd4e9 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetBounded.cpp @@ -14,13 +14,36 @@ struct SliceFromRightConstantOffsetBoundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ssize_t & length, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceFromRightConstantOffsetBounded(source, sink, offset, length); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else + sliceFromRightConstantOffsetBounded(static_cast(source), sink, offset, length); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromRightConstantOffsetBounded(static_cast &>(source), sink, offset, length); + else + sliceFromRightConstantOffsetBounded(source, sink, offset, length); + } } }; diff --git a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp index 945450b4208..59bb0c21599 100644 --- a/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp +++ b/src/Functions/GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp @@ -14,13 +14,36 @@ struct SliceFromRightConstantOffsetUnboundedSelectArraySource : public ArraySourceSelector { template - static void selectImpl(Source && source, size_t & offset, ColumnArray::MutablePtr & result) + static void selectSource(bool is_const, bool is_nullable, Source && source, size_t & offset, ColumnArray::MutablePtr & result) { using SourceType = typename std::decay::type; using Sink = typename SourceType::SinkType; - result = ColumnArray::create(source.createValuesColumn()); - Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); - sliceFromRightConstantOffsetUnbounded(source, sink, offset); + + if (is_nullable) + { + using NullableSource = NullableArraySource; + using NullableSink = typename NullableSource::SinkType; + + auto & nullable_source = static_cast(source); + + result = ColumnArray::create(nullable_source.createValuesColumn()); + NullableSink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else + sliceFromRightConstantOffsetUnbounded(static_cast(source), sink, offset); + } + else + { + result = ColumnArray::create(source.createValuesColumn()); + Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); + + if (is_const) + sliceFromRightConstantOffsetUnbounded(static_cast &>(source), sink, offset); + else + sliceFromRightConstantOffsetUnbounded(source, sink, offset); + } } }; From 13e41cdd20c8204efedcf9c7219443a16915c1e0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 21 Sep 2020 13:24:10 +0300 Subject: [PATCH 234/390] try enable atomic database by default --- docker/test/fasttest/run.sh | 1 + docker/test/stateful/run.sh | 7 +++--- docker/test/stateless/run.sh | 7 +++--- docker/test/stateless_unbundled/run.sh | 7 +++--- docker/test/stress/stress | 2 +- programs/copier/Internals.cpp | 17 +++----------- programs/copier/Internals.h | 2 +- programs/copier/TaskTableAndShard.h | 22 +++++++++++-------- src/Core/Settings.h | 2 +- src/Storages/StorageDictionary.cpp | 8 +++++-- src/Storages/StorageDictionary.h | 3 ++- tests/clickhouse-test | 8 +++---- tests/config/database_atomic_usersd.xml | 1 - tests/config/database_ordinary_usersd.xml | 7 ++++++ .../test_cluster_copier/task0_description.xml | 4 ++-- .../task_month_to_week_description.xml | 4 ++-- .../task_test_block_size.xml | 4 ++-- tests/integration/test_cluster_copier/test.py | 21 ++++++++++-------- .../test_cluster_copier/trivial_test.py | 2 +- .../test_dictionaries_dependency/test.py | 2 +- ...lized_view_and_deduplication_zookeeper.sql | 16 +++++++------- .../00604_show_create_database.reference | 2 +- .../0_stateless/00609_mv_index_in_in.sql | 4 ++-- .../0_stateless/00738_lock_for_inner_table.sh | 4 ++-- .../01190_full_attach_syntax.reference | 18 +++++++-------- .../0_stateless/01190_full_attach_syntax.sql | 16 +++----------- .../01224_no_superfluous_dict_reload.sql | 2 +- ...1225_show_create_table_from_dictionary.sql | 2 +- ...9_bad_arguments_for_bloom_filter.reference | 6 ++--- .../01249_bad_arguments_for_bloom_filter.sql | 6 ++++- ...20_create_sync_race_condition_zookeeper.sh | 9 ++++---- tests/queries/skip_list.json | 16 +++++++------- 32 files changed, 119 insertions(+), 113 deletions(-) create mode 100644 tests/config/database_ordinary_usersd.xml diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ccbadb84f27..560fa9c77af 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -136,6 +136,7 @@ ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/conf ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml # Keep original query_masking_rules.xml diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c3576acc0e4..34980508488 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -23,10 +23,11 @@ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-serv ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ +if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then #FIXME USE_DATABASE_ORDINARY + ln -s /usr/share/clickhouse-test/config/database_ordinary_configd.xml /etc/clickhouse-server/config.d/ fi function start() diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 4a9ad891883..5546f0eef18 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -33,6 +33,8 @@ ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/conf ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ # Retain any pre-existing config and allow ClickHouse to load it if required ln -s --backup=simple --suffix=_original.xml \ @@ -41,9 +43,8 @@ ln -s --backup=simple --suffix=_original.xml \ if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ fi -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ +if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then #FIXME USE_DATABASE_ORDINARY + ln -s /usr/share/clickhouse-test/config/database_ordinary_usersd.xml /etc/clickhouse-server/users.d/ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index 4a9ad891883..13ded2290fb 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -33,6 +33,8 @@ ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/conf ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ +ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ +ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ # Retain any pre-existing config and allow ClickHouse to load it if required ln -s --backup=simple --suffix=_original.xml \ @@ -41,9 +43,8 @@ ln -s --backup=simple --suffix=_original.xml \ if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ fi -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ +if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then #FIXME USE_DATABASE_ORDINARY + ln -s /usr/share/clickhouse-test/config/database_ordinary_configd.xml /etc/clickhouse-server/config.d/ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 60db5ec465c..a36adda3aad 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -29,7 +29,7 @@ def get_options(i): if 0 < i: options += " --order=random" if i % 2 == 1: - options += " --atomic-db-engine" + options += " --db-engine=Ordinary" return options diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 12da07a772a..ca26f0d1831 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -215,31 +215,20 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) return primary_key_columns; } -String extractReplicatedTableZookeeperPath(const ASTPtr & storage_ast) +bool isReplicatedTableEngine(const ASTPtr & storage_ast) { - String storage_str = queryToString(storage_ast); - const auto & storage = storage_ast->as(); const auto & engine = storage.engine->as(); if (!endsWith(engine.name, "MergeTree")) { + String storage_str = queryToString(storage_ast); throw Exception( "Unsupported engine was specified in " + storage_str + ", only *MergeTree engines are supported", ErrorCodes::BAD_ARGUMENTS); } - if (!startsWith(engine.name, "Replicated")) - { - return ""; - } - - auto replicated_table_arguments = engine.arguments->children; - - auto zk_table_path_ast = replicated_table_arguments[0]->as(); - auto zk_table_path_string = zk_table_path_ast.value.safeGet(); - - return zk_table_path_string; + return startsWith(engine.name, "Replicated"); } ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random) diff --git a/programs/copier/Internals.h b/programs/copier/Internals.h index b61b6d59629..7e45c0ea2ee 100644 --- a/programs/copier/Internals.h +++ b/programs/copier/Internals.h @@ -200,7 +200,7 @@ ASTPtr extractOrderBy(const ASTPtr & storage_ast); Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast); -String extractReplicatedTableZookeeperPath(const ASTPtr & storage_ast); +bool isReplicatedTableEngine(const ASTPtr & storage_ast); ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random); diff --git a/programs/copier/TaskTableAndShard.h b/programs/copier/TaskTableAndShard.h index 11ceffd12cd..27c4b89377d 100644 --- a/programs/copier/TaskTableAndShard.h +++ b/programs/copier/TaskTableAndShard.h @@ -48,7 +48,7 @@ struct TaskTable String getCertainPartitionPieceTaskStatusPath(const String & partition_name, const size_t piece_number) const; - bool isReplicatedTable() const { return engine_push_zk_path != ""; } + bool isReplicatedTable() const { return is_replicated_table; } /// Partitions will be split into number-of-splits pieces. /// Each piece will be copied independently. (10 by default) @@ -78,6 +78,7 @@ struct TaskTable /// First argument of Replicated...MergeTree() String engine_push_zk_path; + bool is_replicated_table; ASTPtr rewriteReplicatedCreateQueryToPlain(); @@ -269,7 +270,7 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf engine_push_ast = parseQuery(parser_storage, engine_push_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); primary_key_comma_separated = Nested::createCommaSeparatedStringFrom(extractPrimaryKeyColumnNames(engine_push_ast)); - engine_push_zk_path = extractReplicatedTableZookeeperPath(engine_push_ast); + is_replicated_table = isReplicatedTableEngine(engine_push_ast); } sharding_key_str = config.getString(table_prefix + "sharding_key"); @@ -372,15 +373,18 @@ inline ASTPtr TaskTable::rewriteReplicatedCreateQueryToPlain() auto & new_storage_ast = prev_engine_push_ast->as(); auto & new_engine_ast = new_storage_ast.engine->as(); - auto & replicated_table_arguments = new_engine_ast.arguments->children; - - /// Delete first two arguments of Replicated...MergeTree() table. - replicated_table_arguments.erase(replicated_table_arguments.begin()); - replicated_table_arguments.erase(replicated_table_arguments.begin()); - - /// Remove replicated from name + /// Remove "Replicated" from name new_engine_ast.name = new_engine_ast.name.substr(10); + if (new_engine_ast.arguments) + { + auto & replicated_table_arguments = new_engine_ast.arguments->children; + + /// Delete first two arguments of Replicated...MergeTree() table. + replicated_table_arguments.erase(replicated_table_arguments.begin()); + replicated_table_arguments.erase(replicated_table_arguments.begin()); + } + return new_storage_ast.clone(); } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b96b1b12c24..9449cd571a1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -350,7 +350,7 @@ class IColumn; M(UInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.", 0) \ M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ - M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Ordinary, "Default database engine.", 0) \ + M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \ M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ M(Bool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 5d92b9cec55..99645d09d00 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -100,6 +100,9 @@ StorageDictionary::StorageDictionary( Location location_) : IStorage(table_id_) , dictionary_name(dictionary_name_) + , resolved_dictionary_name(location_ == Location::SameDatabaseAndNameAsDictionary + ? dictionary_name + : DatabaseCatalog::instance().resolveDictionaryName(dictionary_name)) , location(location_) { StorageInMemoryMetadata storage_metadata; @@ -132,7 +135,7 @@ Pipe StorageDictionary::read( const size_t max_block_size, const unsigned /*threads*/) { - auto dictionary = context.getExternalDictionariesLoader().getDictionary(dictionary_name); + auto dictionary = context.getExternalDictionariesLoader().getDictionary(resolved_dictionary_name); auto stream = dictionary->getBlockInputStream(column_names, max_block_size); /// TODO: update dictionary interface for processors. return Pipe(std::make_shared(stream)); @@ -152,7 +155,8 @@ void registerStorageDictionary(StorageFactory & factory) if (!args.attach) { - const auto & dictionary = args.context.getExternalDictionariesLoader().getDictionary(dictionary_name); + auto resolved = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); + const auto & dictionary = args.context.getExternalDictionariesLoader().getDictionary(resolved); const DictionaryStructure & dictionary_structure = dictionary->getStructure(); checkNamesAndTypesCompatibleWithDictionary(dictionary_name, args.columns, dictionary_structure); } diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index d822552124d..528ee6533b7 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -28,7 +28,7 @@ public: static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); static String generateNamesAndTypesDescription(const NamesAndTypesList & list); - const String & dictionaryName() const { return dictionary_name; } + const String & dictionaryName() const { return resolved_dictionary_name; } /// Specifies where the table is located relative to the dictionary. enum class Location @@ -50,6 +50,7 @@ public: private: const String dictionary_name; + const String resolved_dictionary_name; const Location location; protected: diff --git a/tests/clickhouse-test b/tests/clickhouse-test index a3bed189d55..de296cabb7a 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -107,9 +107,9 @@ def remove_control_characters(s): return s def get_db_engine(args): - if args.atomic_db_engine: - return " ENGINE=Atomic" - return "" + if args.db_engine: + return " ENGINE=" + args.db_engine + return "" # Will use default engine def run_single_test(args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file): @@ -792,7 +792,7 @@ if __name__ == '__main__': parser.add_argument('-r', '--server-check-retries', default=30, type=int, help='Num of tries to execute SELECT 1 before tests started') parser.add_argument('--skip-list-path', help="Path to skip-list file") parser.add_argument('--use-skip-list', action='store_true', default=False, help="Use skip list to skip tests if found") - parser.add_argument('--atomic-db-engine', action='store_true', help='Create databases with Atomic engine by default') + parser.add_argument('--db-engine', help='Database engine name') parser.add_argument('--no-stateless', action='store_true', help='Disable all stateless tests') parser.add_argument('--no-stateful', action='store_true', help='Disable all stateful tests') diff --git a/tests/config/database_atomic_usersd.xml b/tests/config/database_atomic_usersd.xml index 201d476da24..58e36cf3884 100644 --- a/tests/config/database_atomic_usersd.xml +++ b/tests/config/database_atomic_usersd.xml @@ -1,7 +1,6 @@ - Atomic 0 diff --git a/tests/config/database_ordinary_usersd.xml b/tests/config/database_ordinary_usersd.xml new file mode 100644 index 00000000000..68f3b044f75 --- /dev/null +++ b/tests/config/database_ordinary_usersd.xml @@ -0,0 +1,7 @@ + + + + Ordinary + + + diff --git a/tests/integration/test_cluster_copier/task0_description.xml b/tests/integration/test_cluster_copier/task0_description.xml index 72eff8d464d..d56053ffd39 100644 --- a/tests/integration/test_cluster_copier/task0_description.xml +++ b/tests/integration/test_cluster_copier/task0_description.xml @@ -33,7 +33,7 @@ 3 4 5 6 1 2 0 - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 + ENGINE=ReplicatedMergeTree PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 d + 1 @@ -93,4 +93,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/task_month_to_week_description.xml b/tests/integration/test_cluster_copier/task_month_to_week_description.xml index ee134603310..26dfc7d3e00 100644 --- a/tests/integration/test_cluster_copier/task_month_to_week_description.xml +++ b/tests/integration/test_cluster_copier/task_month_to_week_description.xml @@ -34,7 +34,7 @@ ENGINE= - ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/b', '{replica}') + ReplicatedMergeTree PARTITION BY toMonday(date) ORDER BY d @@ -97,4 +97,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/task_test_block_size.xml b/tests/integration/test_cluster_copier/task_test_block_size.xml index ea63d580c1c..c9c99a083ea 100644 --- a/tests/integration/test_cluster_copier/task_test_block_size.xml +++ b/tests/integration/test_cluster_copier/task_test_block_size.xml @@ -28,7 +28,7 @@ ENGINE= - ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/test_block_size', '{replica}') + ReplicatedMergeTree ORDER BY d PARTITION BY partition @@ -99,4 +99,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 2a9e696ca46..88dac06f158 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -81,11 +81,11 @@ class Task1: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( cluster_num)) ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + + "ENGINE=ReplicatedMergeTree " + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") @@ -110,10 +110,11 @@ class Task1: class Task2: - def __init__(self, cluster): + def __init__(self, cluster, unique_zk_path): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read() + self.unique_zk_path = unique_zk_path def start(self): instance = cluster.instances['s0_0_0'] @@ -121,11 +122,13 @@ class Task2: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( cluster_num)) ddl_check_query(instance, - "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") + "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/" + self.unique_zk_path + "', " + "'{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") @@ -169,7 +172,7 @@ class Task_test_block_size: ddl_check_query(instance, """ CREATE TABLE test_block_size ON CLUSTER shard_0_0 (partition Date, d UInt64) - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') + ENGINE=ReplicatedMergeTree ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) instance.query( @@ -332,17 +335,17 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs @pytest.mark.timeout(600) def test_copy_month_to_week_partition(started_cluster): - execute_task(Task2(started_cluster), []) + execute_task(Task2(started_cluster, "test1"), []) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering(started_cluster): - execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster, "test2"), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): - execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster, "test3"), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) def test_block_size(started_cluster): diff --git a/tests/integration/test_cluster_copier/trivial_test.py b/tests/integration/test_cluster_copier/trivial_test.py index 3d0c5d0f5b0..035faf0bb9f 100644 --- a/tests/integration/test_cluster_copier/trivial_test.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -59,7 +59,7 @@ class TaskTrivial: for node in [source, destination]: node.query("DROP DATABASE IF EXISTS default") - node.query("CREATE DATABASE IF NOT EXISTS default ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS default") source.query("CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1) " "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index 119bd7c6863..9c36da229e1 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -12,7 +12,7 @@ def start_cluster(): try: cluster.start() for node in nodes: - node.query("CREATE DATABASE IF NOT EXISTS test") + node.query("CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary") node.query("CREATE DATABASE IF NOT EXISTS atest") node.query("CREATE DATABASE IF NOT EXISTS ztest") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") diff --git a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql index 48e1cd65c49..8df012a8588 100644 --- a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql +++ b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql @@ -8,10 +8,10 @@ CREATE TABLE with_deduplication(x UInt32) CREATE TABLE without_deduplication(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00510/without_deduplication', 'r1') ORDER BY x SETTINGS replicated_deduplication_window = 0; -CREATE MATERIALIZED VIEW with_deduplication_mv +CREATE MATERIALIZED VIEW with_deduplication_mv UUID '00000510-1000-4000-8000-000000000001' ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/test_00510/with_deduplication_mv', 'r1') ORDER BY dummy AS SELECT 0 AS dummy, countState(x) AS cnt FROM with_deduplication; -CREATE MATERIALIZED VIEW without_deduplication_mv +CREATE MATERIALIZED VIEW without_deduplication_mv UUID '00000510-1000-4000-8000-000000000002' ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/test_00510/without_deduplication_mv', 'r1') ORDER BY dummy AS SELECT 0 AS dummy, countState(x) AS cnt FROM without_deduplication; @@ -32,12 +32,12 @@ SELECT countMerge(cnt) FROM with_deduplication_mv; SELECT countMerge(cnt) FROM without_deduplication_mv; -- Explicit insert is deduplicated -ALTER TABLE `.inner.with_deduplication_mv` DROP PARTITION ID 'all'; -ALTER TABLE `.inner.without_deduplication_mv` DROP PARTITION ID 'all'; -INSERT INTO `.inner.with_deduplication_mv` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; -INSERT INTO `.inner.with_deduplication_mv` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; -INSERT INTO `.inner.without_deduplication_mv` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; -INSERT INTO `.inner.without_deduplication_mv` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; +ALTER TABLE `.inner_id.00000510-1000-4000-8000-000000000001` DROP PARTITION ID 'all'; +ALTER TABLE `.inner_id.00000510-1000-4000-8000-000000000002` DROP PARTITION ID 'all'; +INSERT INTO `.inner_id.00000510-1000-4000-8000-000000000001` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; +INSERT INTO `.inner_id.00000510-1000-4000-8000-000000000001` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; +INSERT INTO `.inner_id.00000510-1000-4000-8000-000000000002` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; +INSERT INTO `.inner_id.00000510-1000-4000-8000-000000000002` SELECT 0 AS dummy, arrayReduce('countState', [toUInt32(42)]) AS cnt; SELECT ''; SELECT countMerge(cnt) FROM with_deduplication_mv; diff --git a/tests/queries/0_stateless/00604_show_create_database.reference b/tests/queries/0_stateless/00604_show_create_database.reference index a9ad6abea25..c05b088280e 100644 --- a/tests/queries/0_stateless/00604_show_create_database.reference +++ b/tests/queries/0_stateless/00604_show_create_database.reference @@ -1 +1 @@ -CREATE DATABASE test_00604\nENGINE = Ordinary +CREATE DATABASE test_00604\nENGINE = Atomic diff --git a/tests/queries/0_stateless/00609_mv_index_in_in.sql b/tests/queries/0_stateless/00609_mv_index_in_in.sql index 7064d8e36cd..28085194327 100644 --- a/tests/queries/0_stateless/00609_mv_index_in_in.sql +++ b/tests/queries/0_stateless/00609_mv_index_in_in.sql @@ -4,11 +4,11 @@ DROP TABLE IF EXISTS test_mv_00609; create table test_00609 (a Int8) engine=Memory; insert into test_00609 values (1); -create materialized view test_mv_00609 Engine=MergeTree(date, (a), 8192) populate as select a, toDate('2000-01-01') date from test_00609; +create materialized view test_mv_00609 uuid '00000609-1000-4000-8000-000000000001' Engine=MergeTree(date, (a), 8192) populate as select a, toDate('2000-01-01') date from test_00609; select * from test_mv_00609; -- OK select * from test_mv_00609 where a in (select a from test_mv_00609); -- EMPTY (bug) -select * from ".inner.test_mv_00609" where a in (select a from test_mv_00609); -- OK +select * from ".inner_id.00000609-1000-4000-8000-000000000001" where a in (select a from test_mv_00609); -- OK DROP TABLE test_00609; DROP TABLE test_mv_00609; diff --git a/tests/queries/0_stateless/00738_lock_for_inner_table.sh b/tests/queries/0_stateless/00738_lock_for_inner_table.sh index 2f7035b6759..4570c853f31 100755 --- a/tests/queries/0_stateless/00738_lock_for_inner_table.sh +++ b/tests/queries/0_stateless/00738_lock_for_inner_table.sh @@ -7,13 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo "DROP TABLE IF EXISTS tab_00738; DROP TABLE IF EXISTS mv; CREATE TABLE tab_00738(a Int) ENGINE = Log; -CREATE MATERIALIZED VIEW mv ENGINE = Log AS SELECT a FROM tab_00738;" | ${CLICKHOUSE_CLIENT} -n +CREATE MATERIALIZED VIEW mv UUID '00000738-1000-4000-8000-000000000001' ENGINE = Log AS SELECT a FROM tab_00738;" | ${CLICKHOUSE_CLIENT} -n ${CLICKHOUSE_CLIENT} --query_id test_00738 --query "INSERT INTO tab_00738 SELECT number FROM numbers(10000000)" & function drop() { - ${CLICKHOUSE_CLIENT} --query "DROP TABLE \`.inner.mv\`" -n + ${CLICKHOUSE_CLIENT} --query "DROP TABLE \`.inner_id.00000738-1000-4000-8000-000000000001\`" -n } function wait_for_query_to_start() diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.reference b/tests/queries/0_stateless/01190_full_attach_syntax.reference index 619861849c8..4e6eabcd6f0 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.reference +++ b/tests/queries/0_stateless/01190_full_attach_syntax.reference @@ -1,13 +1,13 @@ CREATE DICTIONARY test_01190.dict\n(\n `key` UInt64 DEFAULT 0,\n `col` UInt8 DEFAULT 1\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'test_01190\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) CREATE DICTIONARY test_01190.dict\n(\n `key` UInt64 DEFAULT 0,\n `col` UInt8 DEFAULT 1\n)\nPRIMARY KEY key\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'test_01190\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) -CREATE TABLE default.log\n(\n `s` String\n)\nENGINE = Log -CREATE TABLE default.log\n(\n `s` String\n)\nENGINE = Log() +CREATE TABLE test_01190.log\n(\n `s` String\n)\nENGINE = Log +CREATE TABLE test_01190.log\n(\n `s` String\n)\nENGINE = Log() test -CREATE TABLE default.mt\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date MATERIALIZED \'2000-01-01\'\n)\nENGINE = MergeTree(d, (key, s, n), 1) +CREATE TABLE test_01190.mt\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date MATERIALIZED \'2000-01-01\'\n)\nENGINE = MergeTree(d, (key, s, n), 1) [1,2] Hello 2 -CREATE TABLE default.mt\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date\n)\nENGINE = MergeTree(d, (key, s, n), 1) -CREATE MATERIALIZED VIEW default.mv\n(\n `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM default.log -CREATE MATERIALIZED VIEW default.mv\n(\n `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM default.log -CREATE MATERIALIZED VIEW default.mv\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date\n)\nENGINE = Null AS\nSELECT *\nFROM default.mt -CREATE LIVE VIEW default.lv\n(\n `1` UInt8\n) AS\nSELECT 1 -CREATE LIVE VIEW default.lv\n(\n `1` UInt8\n) AS\nSELECT 1 +CREATE TABLE test_01190.mt\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date\n)\nENGINE = MergeTree(d, (key, s, n), 1) +CREATE MATERIALIZED VIEW test_01190.mv\n(\n `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.log +CREATE MATERIALIZED VIEW test_01190.mv\n(\n `s` String\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.log +CREATE MATERIALIZED VIEW test_01190.mv\n(\n `key` Array(UInt8),\n `s` String,\n `n` UInt64,\n `d` Date\n)\nENGINE = Null AS\nSELECT *\nFROM test_01190.mt +CREATE LIVE VIEW test_01190.lv\n(\n `1` UInt8\n) AS\nSELECT 1 +CREATE LIVE VIEW test_01190.lv\n(\n `1` UInt8\n) AS\nSELECT 1 diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql index 3a91eccc8cd..225e66bed5c 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.sql +++ b/tests/queries/0_stateless/01190_full_attach_syntax.sql @@ -1,5 +1,6 @@ DROP DATABASE IF EXISTS test_01190; -CREATE DATABASE test_01190; +CREATE DATABASE test_01190 ENGINE=Ordinary; +USE test_01190; CREATE TABLE test_01190.table_for_dict (key UInt64, col UInt8) ENGINE = Memory; @@ -14,14 +15,6 @@ ATTACH DICTIONARY test_01190.dict (key UInt64 DEFAULT 0, col UInt8 DEFAULT 42) P ATTACH DICTIONARY test_01190.dict; SHOW CREATE DICTIONARY test_01190.dict; -DROP DATABASE test_01190; - - -DROP TABLE IF EXISTS log; -DROP TABLE IF EXISTS mt; -DROP TABLE IF EXISTS mv; -DROP TABLE IF EXISTS lv; - CREATE TABLE log ENGINE = Log AS SELECT 'test' AS s; SHOW CREATE log; DETACH TABLE log; @@ -58,9 +51,6 @@ DETACH VIEW lv; ATTACH LIVE VIEW lv AS SELECT 1; SHOW CREATE lv; -DROP TABLE log; -DROP TABLE mt; -DROP TABLE mv; -DROP TABLE lv; +DROP DATABASE test_01190; diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql index cf8b2a471c4..55689411ad6 100644 --- a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS dict_db_01224; DROP DATABASE IF EXISTS dict_db_01224_dictionary; -CREATE DATABASE dict_db_01224; +CREATE DATABASE dict_db_01224 ENGINE=Ordinary; CREATE DATABASE dict_db_01224_dictionary Engine=Dictionary; CREATE TABLE dict_db_01224.dict_data (key UInt64, val UInt64) Engine=Memory(); diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql index a494511ebd8..8980a9fd70b 100644 --- a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS dict_db_01225; DROP DATABASE IF EXISTS dict_db_01225_dictionary; -CREATE DATABASE dict_db_01225; +CREATE DATABASE dict_db_01225 ENGINE=Ordinary; CREATE DATABASE dict_db_01225_dictionary Engine=Dictionary; CREATE TABLE dict_db_01225.dict_data (key UInt64, val UInt64) Engine=Memory(); diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference index e3f4955d4cf..fb993e8d572 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.reference @@ -1,3 +1,3 @@ -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(0., 1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 -CREATE TABLE default.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01249.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(0., 1.) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01249.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01249.bloom_filter_idx_good\n(\n `u64` UInt64,\n `i32` Int32,\n `f64` Float64,\n `d` Decimal(10, 2),\n `s` String,\n `e` Enum8(\'a\' = 1, \'b\' = 2, \'c\' = 3),\n `dt` Date,\n INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1\n)\nENGINE = MergeTree()\nORDER BY u64\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql index b60fbc05457..7bf334582a2 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql @@ -1,3 +1,7 @@ +DROP DATABASE IF EXISTS test_01249; +CREATE DATABASE test_01249 ENGINE=Ordinary; +USE test_01249; + CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(0, 1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 42 } CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(-0.1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 36 } CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 36 } @@ -14,4 +18,4 @@ DROP TABLE IF EXISTS bloom_filter_idx_good; ATTACH TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(1.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; SHOW CREATE TABLE bloom_filter_idx_good; -DROP TABLE IF EXISTS bloom_filter_idx_good; +DROP DATABASE test_01249; diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index 5bbec57a236..ce1f08b4c51 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -5,16 +5,17 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS r;" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01320" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" function thread1() { - while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r (x UInt64) ENGINE = ReplicatedMergeTree('/test_01320/table', 'r') ORDER BY x; DROP TABLE r;"; done + while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test_01320/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done } function thread2() { - while true; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA r" 2>/dev/null; done + while true; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA test_01320.r" 2>/dev/null; done } export -f thread1 @@ -25,4 +26,4 @@ timeout 10 bash -c thread2 & wait -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS r;" +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01320" diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index e4713b2d960..4ffa2bd74cb 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -90,17 +90,17 @@ ], "database-atomic": [ /// Inner tables of materialized views have different names - "00738_lock_for_inner_table", - "00609_mv_index_in_in", - "00510_materizlized_view_and_deduplication_zookeeper", + ///"00738_lock_for_inner_table", + ///"00609_mv_index_in_in", + ///"00510_materizlized_view_and_deduplication_zookeeper", /// Different database engine - "00604_show_create_database", + ///"00604_show_create_database", /// UUID must be specified in ATTACH TABLE - "01190_full_attach_syntax", + ///"01190_full_attach_syntax", /// Assumes blocking DROP - "01320_create_sync_race_condition", + ///"01320_create_sync_race_condition", /// Internal distionary name is different - "01225_show_create_table_from_dictionary", - "01224_no_superfluous_dict_reload" + ///"01225_show_create_table_from_dictionary", + ///"01224_no_superfluous_dict_reload" ] } From 7f629c8185a2b8824970158f881bd7f31b8ffbf4 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 21 Sep 2020 14:12:45 +0300 Subject: [PATCH 235/390] Update DatabaseMemory.cpp --- src/Databases/DatabaseMemory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 221e54ce741..5eacb846d52 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -75,7 +75,7 @@ ASTPtr DatabaseMemory::getCreateTableQueryImpl(const String & table_name, const else return {}; } - return it->second; + return it->second->clone(); } UUID DatabaseMemory::tryGetTableUUID(const String & table_name) const From 2f8d7ae208f1ffc8db62256583dc48078823c6b6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 21 Sep 2020 15:01:27 +0300 Subject: [PATCH 236/390] fix the conditions --- docker/test/performance-comparison/compare.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 8585603b010..df9619c28c7 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -155,13 +155,13 @@ function run_tests # are already set, keep those values. if [ "$PR_TO_TEST" -ne 0 ] \ && [ "$(wc -l < changed-test-definitions.txt)" -eq 0 ] \ - && [ "$(wc -l < changed-test-files.txt)" -eq 0 ] + && [ "$(wc -l < changed-test-scripts.txt)" -eq 0 ] then - CHPC_RUNS=${CHPC_RUNS:-13} - CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-0} - else CHPC_RUNS=${CHPC_RUNS:-7} CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-20} + else + CHPC_RUNS=${CHPC_RUNS:-13} + CHPC_MAX_QUERIES=${CHPC_MAX_QUERIES:-0} fi export CHPC_RUNS export CHPC_MAX_QUERIES From 4dae587e2d3a78208f42c508bebe9b5a36126ada Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 21 Sep 2020 15:02:34 +0300 Subject: [PATCH 237/390] fix test --- tests/performance/columns_hashing.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index 271fff6e543..c45e5d1fe39 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -7,6 +7,6 @@ select sum(UserID + 1 in (select UserID from hits_100m_single)) from hits_100m_single select sum((UserID + 1, RegionID) in (select UserID, RegionID from hits_10m_single)) from hits_10m_single select sum(URL in (select URL from hits_10m_single where URL != '')) from hits_10m_single - select sum(MobilePhoneModel in (select MobilePhoneModel from hits_10m_single where MobilePhoneModel != '')) from hits_10m_single + select sum(MobilePhoneModel in (select MobilePhoneModel from hits_100m_single where MobilePhoneModel != '')) from hits_100m_single select sum((MobilePhoneModel, UserID + 1) in (select MobilePhoneModel, UserID from hits_100m_single where MobilePhoneModel != '')) from hits_100m_single From 679c5278e49cd0050ad75d4e7f772cdf89263a95 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 21 Sep 2020 15:14:47 +0300 Subject: [PATCH 238/390] reword the diagnostic about short queries in perf test --- docker/test/performance-comparison/README.md | 2 +- docker/test/performance-comparison/compare.sh | 9 +++++---- docker/test/performance-comparison/report.py | 14 +++++++------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docker/test/performance-comparison/README.md b/docker/test/performance-comparison/README.md index ec5691488f0..3953e99fc0f 100644 --- a/docker/test/performance-comparison/README.md +++ b/docker/test/performance-comparison/README.md @@ -45,7 +45,7 @@ Action required for every item -- these are errors that must be fixed. This table shows queries that take significantly longer to process on the client than on the server. A possible reason might be sending too much data to the client, e.g., a forgotten `format Null`. -#### Inconsistent Short Marking +#### Unexpected Query Duration Action required for every item -- these are errors that must be fixed. Queries that have "short" duration (on the order of 0.1 s) can't be reliably tested in a normal way, where we perform a small (about ten) measurements for each server, because the signal-to-noise ratio is much smaller. There is a special mode for such queries that instead runs them for a fixed amount of time, normally with much higher number of measurements (up to thousands). This mode must be explicitly enabled by the test author to avoid accidental errors. It must be used only for queries that are meant to complete "immediately", such as `select count(*)`. If your query is not supposed to be "immediate", try to make it run longer, by e.g. processing more data. diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index f90020f80eb..6a9898ba797 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -688,12 +688,13 @@ create view shortness -- Report of queries that have inconsistent 'short' markings: -- 1) have short duration, but are not marked as 'short' -- 2) the reverse -- marked 'short' but take too long. --- The threshold for 2) is twice the threshold for 1), to avoid jitter. +-- The threshold for 2) is significantly larger than the threshold for 1), to +-- avoid jitter. create table inconsistent_short_marking_report - engine File(TSV, 'report/inconsistent-short-marking.tsv') + engine File(TSV, 'report/unexpected-query-duration.tsv') as select - multiIf(marked_short and time > 0.1, 'marked as short but is too long', - not marked_short and time < 0.02, 'is short but not marked as such', + multiIf(marked_short and time > 0.1, '"short" queries must run faster than 0.02 s', + not marked_short and time < 0.02, '"normal" queries must run longer than 0.1 s', '') problem, marked_short, time, test, query_index, query_display_name diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index e8481f77340..8304aa55fc2 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -98,9 +98,9 @@ th {{ tr:nth-child(odd) td {{filter: brightness(90%);}} -.inconsistent-short-marking tr :nth-child(2), -.inconsistent-short-marking tr :nth-child(3), -.inconsistent-short-marking tr :nth-child(5), +.unexpected-query-duration tr :nth-child(2), +.unexpected-query-duration tr :nth-child(3), +.unexpected-query-duration tr :nth-child(5), .all-query-times tr :nth-child(1), .all-query-times tr :nth-child(2), .all-query-times tr :nth-child(3), @@ -325,13 +325,13 @@ if args.report == 'main': if slow_on_client_rows: errors_explained.append([f'Some queries are taking noticeable time client-side (missing `FORMAT Null`?)']); - unmarked_short_rows = tsvRows('report/inconsistent-short-marking.tsv') + unmarked_short_rows = tsvRows('report/unexpected-query-duration.tsv') error_tests += len(unmarked_short_rows) - addSimpleTable('Inconsistent Short Marking', - ['Problem', 'Is marked as short', 'New client time, s', 'Test', '#', 'Query'], + addSimpleTable('Unexpected Query Duration', + ['Problem', 'Marked as "short"?', 'Run time, s', 'Test', '#', 'Query'], unmarked_short_rows) if unmarked_short_rows: - errors_explained.append([f'Some queries have inconsistent short marking']); + errors_explained.append([f'Some queries have unexpected duration']); def add_partial(): rows = tsvRows('report/partial-queries-report.tsv') From fd2cc343e6fd84c1a1e3da79173acc0ea361819b Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 21 Sep 2020 15:37:34 +0300 Subject: [PATCH 239/390] Update renameat2.cpp --- src/Common/renameat2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 02acd8331d2..24e414122dc 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -81,7 +81,7 @@ static bool renameat2(const std::string & old_path, const std::string & new_path static bool renameat2(const std::string &, const std::string &, int) { - return false + return false; } #endif From 741912cafd2814088202b4ca740bcff25bf2cf7c Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 21 Sep 2020 21:03:17 +0800 Subject: [PATCH 240/390] Don't hide interface symbols --- src/Functions/GatherUtils/IArraySink.h | 3 ++- src/Functions/GatherUtils/IArraySource.h | 3 ++- src/Functions/GatherUtils/IValueSource.h | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Functions/GatherUtils/IArraySink.h b/src/Functions/GatherUtils/IArraySink.h index a41d99b5f6a..8bc3f05edd1 100644 --- a/src/Functions/GatherUtils/IArraySink.h +++ b/src/Functions/GatherUtils/IArraySink.h @@ -13,7 +13,6 @@ namespace ErrorCodes namespace GatherUtils { -#pragma GCC visibility push(hidden) struct IArraySink { @@ -25,6 +24,8 @@ struct IArraySink } }; +#pragma GCC visibility push(hidden) + template class ArraySinkImpl : public Visitable {}; diff --git a/src/Functions/GatherUtils/IArraySource.h b/src/Functions/GatherUtils/IArraySource.h index 386108f038e..c34c3ddb683 100644 --- a/src/Functions/GatherUtils/IArraySource.h +++ b/src/Functions/GatherUtils/IArraySource.h @@ -13,7 +13,6 @@ namespace ErrorCodes namespace GatherUtils { -#pragma GCC visibility push(hidden) struct IArraySource { @@ -31,6 +30,8 @@ struct IArraySource } }; +#pragma GCC visibility push(hidden) + template class ArraySourceImpl : public Visitable {}; diff --git a/src/Functions/GatherUtils/IValueSource.h b/src/Functions/GatherUtils/IValueSource.h index 55af364a3a9..2be44143c84 100644 --- a/src/Functions/GatherUtils/IValueSource.h +++ b/src/Functions/GatherUtils/IValueSource.h @@ -13,7 +13,6 @@ namespace ErrorCodes namespace GatherUtils { -#pragma GCC visibility push(hidden) struct IValueSource { @@ -27,6 +26,8 @@ struct IValueSource virtual bool isConst() const { return false; } }; +#pragma GCC visibility push(hidden) + template class ValueSourceImpl : public Visitable {}; From 494e59e7c12e0887a32b2564463a84aea500118b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Mon, 21 Sep 2020 19:12:31 +0300 Subject: [PATCH 241/390] DOCKER_KERBEROS_KDC_TAG --- .../runner/compose/docker_compose_kerberized_kafka.yml | 3 +-- docker/test/integration/runner/dockerd-entrypoint.sh | 1 + tests/integration/runner | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 34c453370e9..3ce0000b148 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -50,8 +50,7 @@ services: - label:disable kafka_kerberos: - # build: ${KERBERIZED_KAFKA_DIR} - image: yandex/clickhouse-kerberos-kdc:latest + image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG} hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index c38260279ed..cbdb7317b1e 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -27,6 +27,7 @@ export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest} export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest} export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest} export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest} +export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest} cd /ClickHouse/tests/integration exec "$@" diff --git a/tests/integration/runner b/tests/integration/runner index f097a42e52a..f38ab0aa042 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -156,6 +156,8 @@ if __name__ == "__main__": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) elif image == "yandex/clickhouse-integration-test": env_tags += "-e {}={}".format("DOCKER_BASE_TAG", tag) + elif image == "yandex/clickhouse-kerberos-kdc": + env_tags += "-e {}={}".format("DOCKER_KERBEROS_KDC_TAG", tag) else: logging.info("Unknown image {}".format(image)) From c77a6967a3a9615b372d4f1d0d5c07ec5fa850a2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 21 Sep 2020 19:22:23 +0300 Subject: [PATCH 242/390] Remove commented code. --- .../GatherUtils/ArraySourceVisitor.h | 4 ---- src/Functions/GatherUtils/concat.cpp | 22 ------------------- 2 files changed, 26 deletions(-) diff --git a/src/Functions/GatherUtils/ArraySourceVisitor.h b/src/Functions/GatherUtils/ArraySourceVisitor.h index 0446839325e..260e9a87161 100644 --- a/src/Functions/GatherUtils/ArraySourceVisitor.h +++ b/src/Functions/GatherUtils/ArraySourceVisitor.h @@ -19,10 +19,6 @@ struct ConstSource; using NumericArraySources = typename TypeListMap::Type; using BasicArraySources = typename AppendToTypeList::Type; -//using NullableArraySources = typename TypeListMap::Type; -//using BasicAndNullableArraySources = typename TypeListConcat::Type; -//using ConstArraySources = typename TypeListMap::Type; -//using TypeListArraySources = typename TypeListConcat::Type; class ArraySourceVisitor : public ApplyTypeListForClass::Type { diff --git a/src/Functions/GatherUtils/concat.cpp b/src/Functions/GatherUtils/concat.cpp index 4fe6948ab56..d73c98a0e88 100644 --- a/src/Functions/GatherUtils/concat.cpp +++ b/src/Functions/GatherUtils/concat.cpp @@ -23,28 +23,6 @@ struct ArrayConcat : public ArraySourceSelector { using Sources = std::vector>; -// template -// static void selectSource(Source && source, const Sources & sources, ColumnArray::MutablePtr & result) -// { -// using SourceType = typename std::decay::type; -// using Sink = typename SourceType::SinkType; -// result = ColumnArray::create(source.createValuesColumn()); -// Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); -// -// concat(sources, std::move(sink)); -// } -// -// template -// static void selectImpl(ConstSource && source, const Sources & sources, ColumnArray::MutablePtr & result) -// { -// using SourceType = typename std::decay::type; -// using Sink = typename SourceType::SinkType; -// result = ColumnArray::create(source.createValuesColumn()); -// Sink sink(result->getData(), result->getOffsets(), source.getColumnSize()); -// -// concat(sources, std::move(sink)); -// } - template static void selectSource(bool /*is_const*/, bool is_nullable, Source & source, const Sources & sources, ColumnArray::MutablePtr & result) { From 10d3a1d730f6f08fe9f1d8403a209980f7849408 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 21 Sep 2020 20:49:15 +0300 Subject: [PATCH 243/390] Update ThreadPool.cpp --- src/Common/ThreadPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index f1fa82b7e68..56198b97be5 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -233,7 +233,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::is_same_v ? CurrentMetrics::GlobalThreadActive : CurrentMetrics::LocalThreadActive); job(); - job = Job(); + job = {}; } catch (...) { From 6e596b9c7d5673372d3bf1eeed5b6e4df6194e0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Sep 2020 21:15:41 +0300 Subject: [PATCH 244/390] Added a test for Enum conversion in Distributed table if one shard is local --- ...buted_local_type_conversion_enum.reference | 21 +++++++++++++++++ ...distributed_local_type_conversion_enum.sql | 23 +++++++++++++++++++ .../queries/0_stateless/arcadia_skip_list.txt | 1 + 3 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.reference create mode 100644 tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.sql diff --git a/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.reference b/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.reference new file mode 100644 index 00000000000..94a9822a8df --- /dev/null +++ b/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.reference @@ -0,0 +1,21 @@ +abc +def +ghi +--- +abc +def +xyz +--- +abc +def +xyz +abc +def +xyz +--- +abc +def +xyz +abc +def +xyz diff --git a/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.sql b/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.sql new file mode 100644 index 00000000000..6f62716d312 --- /dev/null +++ b/tests/queries/0_stateless/01505_distributed_local_type_conversion_enum.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS d; + +CREATE TABLE t (x Enum8('abc' = 0, 'def' = 1, 'ghi' = 2)) ENGINE = TinyLog; +INSERT INTO t VALUES (0), (1), (2); +SELECT * FROM t; + +SELECT '---'; +CREATE TABLE d (x Enum8('abc' = 0, 'def' = 1, 'xyz' = 2)) ENGINE = Distributed(test_shard_localhost, currentDatabase(), t); +SELECT * FROM d; +DROP TABLE d; + +SELECT '---'; +CREATE TABLE d (x Enum8('abc' = 0, 'def' = 1, 'xyz' = 2)) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t); +SELECT * FROM d; +DROP TABLE d; + +SELECT '---'; +CREATE TABLE d (x Enum8('abc' = 0, 'def' = 1, 'xyz' = 2)) ENGINE = Distributed(test_cluster_two_shards_localhost, currentDatabase(), t); +SELECT * FROM d; +DROP TABLE d; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index d776f8d80d6..69391ca9fd4 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -144,3 +144,4 @@ 01473_event_time_microseconds 01461_query_start_time_microseconds 01455_shard_leaf_max_rows_bytes_to_read +01505_distributed_local_type_conversion_enum From 4696e2279971b0050ff256187305be15050412fb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 21 Sep 2020 22:08:41 +0300 Subject: [PATCH 245/390] Update perf.py --- docker/test/performance-comparison/perf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index a37ef553cfb..79cdc8ea8d2 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -277,7 +277,7 @@ for query_index in queries_to_run: # Stop processing pathologically slow queries, to avoid timing out # the entire test task. This shouldn't really happen, so we don't # need much handling for this case and can just exit. - print(f'The query no. {query_index} is taking too long to run ({c.elapsed} s)', file=sys.stderr) + print(f'The query no. {query_index} is taking too long to run ({elapsed} s)', file=sys.stderr) exit(2) # Be careful with the counter, after this line it's the next iteration From d87f3660ee2d75c993bc2b6356dae37a9186e6e3 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 21 Sep 2020 23:19:46 +0300 Subject: [PATCH 246/390] done --- .../test.py | 346 ------------------ .../__init__.py | 0 .../common.py | 239 ++++++++++++ .../configs/config.xml | 0 .../configs/dictionaries/.gitkeep | 0 .../configs/disable_ssl_verification.xml | 0 .../configs/enable_dictionaries.xml | 0 .../configs/users.xml | 0 .../test_cassandra.py | 82 +++++ .../test_clickhouse_local.py | 82 +++++ .../test_clickhouse_remote.py | 84 +++++ .../test_executable_cache.py | 82 +++++ .../test_executable_hashed.py | 82 +++++ .../test_file.py | 82 +++++ .../test_http.py | 84 +++++ .../test_https.py | 84 +++++ .../test_mongo.py | 82 +++++ .../test_mongo_uri.py | 82 +++++ .../test_mysql.py | 82 +++++ 19 files changed, 1147 insertions(+), 346 deletions(-) delete mode 100644 tests/integration/test_dictionaries_all_layouts_and_sources/test.py rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/__init__.py (100%) create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/common.py rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/configs/config.xml (100%) rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/configs/dictionaries/.gitkeep (100%) rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/configs/disable_ssl_verification.xml (100%) rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/configs/enable_dictionaries.xml (100%) rename tests/integration/{test_dictionaries_all_layouts_and_sources => test_dictionaries_all_layouts_separate_sources}/configs/users.xml (100%) create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py create mode 100644 tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/tests/integration/test_dictionaries_all_layouts_and_sources/test.py deleted file mode 100644 index 5880ead7c5a..00000000000 --- a/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ /dev/null @@ -1,346 +0,0 @@ -import math -import os - -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout -from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceCassandra -from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, \ - SourceExecutableHashed - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') - -FIELDS = { - "simple": [ - Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999), - Field("UInt8_", 'UInt8', default_value_for_get=55), - Field("UInt16_", 'UInt16', default_value_for_get=66), - Field("UInt32_", 'UInt32', default_value_for_get=77), - Field("UInt64_", 'UInt64', default_value_for_get=88), - Field("Int8_", 'Int8', default_value_for_get=-55), - Field("Int16_", 'Int16', default_value_for_get=-66), - Field("Int32_", 'Int32', default_value_for_get=-77), - Field("Int64_", 'Int64', default_value_for_get=-88), - Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), - Field("Date_", 'Date', default_value_for_get='2018-12-30'), - Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), - Field("String_", 'String', default_value_for_get='hi'), - Field("Float32_", 'Float32', default_value_for_get=555.11), - Field("Float64_", 'Float64', default_value_for_get=777.11), - Field("ParentKeyField", "UInt64", default_value_for_get=444, hierarchical=True) - ], - "complex": [ - Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999), - Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx'), - Field("UInt8_", 'UInt8', default_value_for_get=55), - Field("UInt16_", 'UInt16', default_value_for_get=66), - Field("UInt32_", 'UInt32', default_value_for_get=77), - Field("UInt64_", 'UInt64', default_value_for_get=88), - Field("Int8_", 'Int8', default_value_for_get=-55), - Field("Int16_", 'Int16', default_value_for_get=-66), - Field("Int32_", 'Int32', default_value_for_get=-77), - Field("Int64_", 'Int64', default_value_for_get=-88), - Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), - Field("Date_", 'Date', default_value_for_get='2018-12-30'), - Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), - Field("String_", 'String', default_value_for_get='hi'), - Field("Float32_", 'Float32', default_value_for_get=555.11), - Field("Float64_", 'Float64', default_value_for_get=777.11), - ], - "ranged": [ - Field("KeyField1", 'UInt64', is_key=True), - Field("KeyField2", 'Date', is_range_key=True), - Field("StartDate", 'Date', range_hash_type='min'), - Field("EndDate", 'Date', range_hash_type='max'), - Field("UInt8_", 'UInt8', default_value_for_get=55), - Field("UInt16_", 'UInt16', default_value_for_get=66), - Field("UInt32_", 'UInt32', default_value_for_get=77), - Field("UInt64_", 'UInt64', default_value_for_get=88), - Field("Int8_", 'Int8', default_value_for_get=-55), - Field("Int16_", 'Int16', default_value_for_get=-66), - Field("Int32_", 'Int32', default_value_for_get=-77), - Field("Int64_", 'Int64', default_value_for_get=-88), - Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), - Field("Date_", 'Date', default_value_for_get='2018-12-30'), - Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), - Field("String_", 'String', default_value_for_get='hi'), - Field("Float32_", 'Float32', default_value_for_get=555.11), - Field("Float64_", 'Float64', default_value_for_get=777.11), - ] -} - -VALUES = { - "simple": [ - [1, 22, 333, 4444, 55555, -6, -77, - -888, -999, '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', 'hello', 22.543, 3332154213.4, 0], - [2, 3, 4, 5, 6, -7, -8, - -9, -10, '550e8400-e29b-41d4-a716-446655440002', - '1978-06-28', '1986-02-28 23:42:25', 'hello', 21.543, 3222154213.4, 1] - ], - "complex": [ - [1, 'world', 22, 333, 4444, 55555, -6, - -77, -888, -999, '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', - 'hello', 22.543, 3332154213.4], - [2, 'qwerty2', 52, 2345, 6544, 9191991, -2, - -717, -81818, -92929, '550e8400-e29b-41d4-a716-446655440007', - '1975-09-28', '2000-02-28 23:33:24', - 'my', 255.543, 3332221.44] - - ], - "ranged": [ - [1, '2019-02-10', '2019-02-01', '2019-02-28', - 22, 333, 4444, 55555, -6, -77, -888, -999, - '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', 'hello', - 22.543, 3332154213.4], - [2, '2019-04-10', '2019-04-01', '2019-04-28', - 11, 3223, 41444, 52515, -65, -747, -8388, -9099, - '550e8400-e29b-41d4-a716-446655440004', - '1973-06-29', '2002-02-28 23:23:25', '!!!!', - 32.543, 3332543.4] - ] -} - -LAYOUTS = [ - Layout("flat"), - Layout("hashed"), - Layout("cache"), - Layout("complex_key_hashed"), - Layout("complex_key_cache"), - Layout("range_hashed"), - Layout("direct"), - Layout("complex_key_direct") -] - -SOURCES = [ - SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", ""), - SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), - SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"), - SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"), - SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", ""), - SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", ""), - SourceFile("File", "localhost", "9000", "node", "9000", "", ""), - SourceExecutableHashed("ExecutableHashed", "localhost", "9000", "node", "9000", "", ""), - SourceExecutableCache("ExecutableCache", "localhost", "9000", "node", "9000", "", ""), - SourceHTTP("SourceHTTP", "localhost", "9000", "clickhouse1", "9000", "", ""), - SourceHTTPS("SourceHTTPS", "localhost", "9000", "clickhouse1", "9000", "", ""), -] - -DICTIONARIES = [] - -cluster = None -node = None - - -def get_dict(source, layout, fields, suffix_name=''): - global dict_configs_path - - structure = DictionaryStructure(layout, fields) - dict_name = source.name + "_" + layout.name + '_' + suffix_name - dict_path = os.path.join(dict_configs_path, dict_name + '.xml') - dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields) - dictionary.generate_config() - return dictionary - - -def setup_module(module): - global DICTIONARIES - global cluster - global node - global dict_configs_path - - for f in os.listdir(dict_configs_path): - os.remove(os.path.join(dict_configs_path, f)) - - for layout in LAYOUTS: - for source in SOURCES: - if source.compatible_with_layout(layout): - DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type])) - else: - print "Source", source.name, "incompatible with layout", layout.name - - cluster = ClickHouseCluster(__file__) - - main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) - - cluster.add_instance('clickhouse1', main_configs=main_configs) - - dictionaries = [] - for fname in os.listdir(dict_configs_path): - dictionaries.append(os.path.join(dict_configs_path, fname)) - - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True, - with_mongo=True, with_redis=True, with_cassandra=True) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - for dictionary in DICTIONARIES: - print "Preparing", dictionary.name - dictionary.prepare_source(cluster) - print "Prepared" - - yield cluster - - finally: - cluster.shutdown() - - -def get_dictionaries(fold, total_folds, all_dicts): - chunk_len = int(math.ceil(len(all_dicts) / float(total_folds))) - if chunk_len * fold >= len(all_dicts): - return [] - return all_dicts[fold * chunk_len: (fold + 1) * chunk_len] - - -def remove_mysql_dicts(): - """ - We have false-positive race condition in our openSSL version. - MySQL dictionary use OpenSSL, so to prevent known failure we - disable tests for these dictionaries. - - Read of size 8 at 0x7b3c00005dd0 by thread T61 (mutexes: write M1010349240585225536): - #0 EVP_CIPHER_mode (clickhouse+0x13b2223b) - #1 do_ssl3_write (clickhouse+0x13a137bc) - #2 ssl3_write_bytes (clickhouse+0x13a12387) - #3 ssl3_write (clickhouse+0x139db0e6) - #4 ssl_write_internal (clickhouse+0x139eddce) - #5 SSL_write (clickhouse+0x139edf20) - #6 ma_tls_write (clickhouse+0x139c7557) - #7 ma_pvio_tls_write (clickhouse+0x139a8f59) - #8 ma_pvio_write (clickhouse+0x139a8488) - #9 ma_net_real_write (clickhouse+0x139a4e2c) - #10 ma_net_write_command (clickhouse+0x139a546d) - #11 mthd_my_send_cmd (clickhouse+0x13992546) - #12 mysql_close_slow_part (clickhouse+0x13999afd) - #13 mysql_close (clickhouse+0x13999071) - #14 mysqlxx::Connection::~Connection() (clickhouse+0x1370f814) - #15 mysqlxx::Pool::~Pool() (clickhouse+0x13715a7b) - - TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed - """ - - # global DICTIONARIES - # DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")] - - -@pytest.mark.parametrize("fold", list(range(10))) -def test_simple_dictionaries(started_cluster, fold): - if node.is_built_with_thread_sanitizer(): - remove_mysql_dicts() - - fields = FIELDS["simple"] - values = VALUES["simple"] - data = [Row(fields, vals) for vals in values] - - all_simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"] - simple_dicts = get_dictionaries(fold, 10, all_simple_dicts) - - print "Length of dicts:", len(simple_dicts) - for dct in simple_dicts: - dct.load_data(data) - - node.query("system reload dictionaries") - - queries_with_answers = [] - for dct in simple_dicts: - for row in data: - for field in fields: - if not field.is_key: - for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) - - for query in dct.get_select_has_queries(field, row): - queries_with_answers.append((query, 1)) - - for query in dct.get_select_get_or_default_queries(field, row): - queries_with_answers.append((query, field.default_value_for_get)) - for query in dct.get_hierarchical_queries(data[0]): - queries_with_answers.append((query, [1])) - - for query in dct.get_hierarchical_queries(data[1]): - queries_with_answers.append((query, [2, 1])) - - for query in dct.get_is_in_queries(data[0], data[1]): - queries_with_answers.append((query, 0)) - - for query in dct.get_is_in_queries(data[1], data[0]): - queries_with_answers.append((query, 1)) - - for query, answer in queries_with_answers: - print query - if isinstance(answer, list): - answer = str(answer).replace(' ', '') - assert node.query(query) == str(answer) + '\n' - - -@pytest.mark.parametrize("fold", list(range(10))) -def test_complex_dictionaries(started_cluster, fold): - if node.is_built_with_thread_sanitizer(): - remove_mysql_dicts() - - fields = FIELDS["complex"] - values = VALUES["complex"] - data = [Row(fields, vals) for vals in values] - - all_complex_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "complex"] - complex_dicts = get_dictionaries(fold, 10, all_complex_dicts) - - for dct in complex_dicts: - dct.load_data(data) - - node.query("system reload dictionaries") - - queries_with_answers = [] - for dct in complex_dicts: - for row in data: - for field in fields: - if not field.is_key: - for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) - - for query in dct.get_select_has_queries(field, row): - queries_with_answers.append((query, 1)) - - for query in dct.get_select_get_or_default_queries(field, row): - queries_with_answers.append((query, field.default_value_for_get)) - - for query, answer in queries_with_answers: - print query - assert node.query(query) == str(answer) + '\n' - - -@pytest.mark.parametrize("fold", list(range(10))) -def test_ranged_dictionaries(started_cluster, fold): - if node.is_built_with_thread_sanitizer(): - remove_mysql_dicts() - - fields = FIELDS["ranged"] - values = VALUES["ranged"] - data = [Row(fields, vals) for vals in values] - - all_ranged_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "ranged"] - ranged_dicts = get_dictionaries(fold, 10, all_ranged_dicts) - - for dct in ranged_dicts: - dct.load_data(data) - - node.query("system reload dictionaries") - - queries_with_answers = [] - for dct in ranged_dicts: - for row in data: - for field in fields: - if not field.is_key and not field.is_range: - for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) - - for query, answer in queries_with_answers: - print query - assert node.query(query) == str(answer) + '\n' diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/__init__.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/__init__.py similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/__init__.py rename to tests/integration/test_dictionaries_all_layouts_separate_sources/__init__.py diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py new file mode 100644 index 00000000000..0411b5d9475 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py @@ -0,0 +1,239 @@ +import os + +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout + +KEY_FIELDS = { + "simple": [ + Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999) + ], + "complex": [ + Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999), + Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx') + ], + "ranged": [ + Field("KeyField1", 'UInt64', is_key=True), + Field("KeyField2", 'Date', is_range_key=True) + ] +} + +START_FIELDS = { + "simple": [], + "complex": [], + "ranged" : [ + Field("StartDate", 'Date', range_hash_type='min'), + Field("EndDate", 'Date', range_hash_type='max') + ] +} + +MIDDLE_FIELDS = [ + Field("UInt8_", 'UInt8', default_value_for_get=55), + Field("UInt16_", 'UInt16', default_value_for_get=66), + Field("UInt32_", 'UInt32', default_value_for_get=77), + Field("UInt64_", 'UInt64', default_value_for_get=88), + Field("Int8_", 'Int8', default_value_for_get=-55), + Field("Int16_", 'Int16', default_value_for_get=-66), + Field("Int32_", 'Int32', default_value_for_get=-77), + Field("Int64_", 'Int64', default_value_for_get=-88), + Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), + Field("Date_", 'Date', default_value_for_get='2018-12-30'), + Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), + Field("String_", 'String', default_value_for_get='hi'), + Field("Float32_", 'Float32', default_value_for_get=555.11), + Field("Float64_", 'Float64', default_value_for_get=777.11), +] + +END_FIELDS = { + "simple" : [ + Field("ParentKeyField", "UInt64", default_value_for_get=444, hierarchical=True) + ], + "complex" : [], + "ranged" : [] +} + +LAYOUTS_SIMPLE = ["flat", "hashed", "cache", "direct"] +LAYOUTS_COMPLEX = ["complex_key_hashed", "complex_key_cache", "complex_key_direct"] +LAYOUTS_RANGED = ["range_hashed"] + +VALUES = { + "simple": [ + [1, 22, 333, 4444, 55555, -6, -77, + -888, -999, '550e8400-e29b-41d4-a716-446655440003', + '1973-06-28', '1985-02-28 23:43:25', 'hello', 22.543, 3332154213.4, 0], + [2, 3, 4, 5, 6, -7, -8, + -9, -10, '550e8400-e29b-41d4-a716-446655440002', + '1978-06-28', '1986-02-28 23:42:25', 'hello', 21.543, 3222154213.4, 1] + ], + "complex": [ + [1, 'world', 22, 333, 4444, 55555, -6, + -77, -888, -999, '550e8400-e29b-41d4-a716-446655440003', + '1973-06-28', '1985-02-28 23:43:25', + 'hello', 22.543, 3332154213.4], + [2, 'qwerty2', 52, 2345, 6544, 9191991, -2, + -717, -81818, -92929, '550e8400-e29b-41d4-a716-446655440007', + '1975-09-28', '2000-02-28 23:33:24', + 'my', 255.543, 3332221.44] + ], + "ranged": [ + [1, '2019-02-10', '2019-02-01', '2019-02-28', + 22, 333, 4444, 55555, -6, -77, -888, -999, + '550e8400-e29b-41d4-a716-446655440003', + '1973-06-28', '1985-02-28 23:43:25', 'hello', + 22.543, 3332154213.4], + [2, '2019-04-10', '2019-04-01', '2019-04-28', + 11, 3223, 41444, 52515, -65, -747, -8388, -9099, + '550e8400-e29b-41d4-a716-446655440004', + '1973-06-29', '2002-02-28 23:23:25', '!!!!', + 32.543, 3332543.4] + ] +} + + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +DICT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'configs/dictionaries') + +def get_dict(source, layout, fields, suffix_name=''): + global DICT_CONFIG_PATH + structure = DictionaryStructure(layout, fields) + dict_name = source.name + "_" + layout.name + '_' + suffix_name + dict_path = os.path.join(DICT_CONFIG_PATH, dict_name + '.xml') + dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields) + dictionary.generate_config() + return dictionary + +class SimpleLayoutTester: + def __init__(self): + self.fields = KEY_FIELDS["simple"] + START_FIELDS["simple"] + MIDDLE_FIELDS + END_FIELDS["simple"] + self.values = VALUES["simple"] + self.data = [Row(self.fields, vals) for vals in self.values] + self.layout_to_dictionary = dict() + + def create_dictionaries(self, source_): + for layout in LAYOUTS_SIMPLE: + if source_.compatible_with_layout(Layout(layout)): + self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields) + + def prepare(self, cluster_): + for _, dictionary in self.layout_to_dictionary.items(): + dictionary.prepare_source(cluster_) + dictionary.load_data(self.data) + + def execute(self, layout_name, node): + if not self.layout_to_dictionary.has_key(layout_name): + raise RuntimeError("Source doesn't support layout: {}".format(layout_name)) + + dct = self.layout_to_dictionary[layout_name] + + node.query("system reload dictionaries") + queries_with_answers = [] + + for row in self.data: + for field in self.fields: + if not field.is_key: + for query in dct.get_select_get_queries(field, row): + queries_with_answers.append((query, row.get_value_by_name(field.name))) + + for query in dct.get_select_has_queries(field, row): + queries_with_answers.append((query, 1)) + + for query in dct.get_select_get_or_default_queries(field, row): + queries_with_answers.append((query, field.default_value_for_get)) + + for query in dct.get_hierarchical_queries(self.data[0]): + queries_with_answers.append((query, [1])) + + for query in dct.get_hierarchical_queries(self.data[1]): + queries_with_answers.append((query, [2, 1])) + + for query in dct.get_is_in_queries(self.data[0], self.data[1]): + queries_with_answers.append((query, 0)) + + for query in dct.get_is_in_queries(self.data[1], self.data[0]): + queries_with_answers.append((query, 1)) + + for query, answer in queries_with_answers: + # print query + if isinstance(answer, list): + answer = str(answer).replace(' ', '') + assert node.query(query) == str(answer) + '\n' + + +class ComplexLayoutTester: + def __init__(self): + self.fields = KEY_FIELDS["complex"] + START_FIELDS["complex"] + MIDDLE_FIELDS + END_FIELDS["complex"] + self.values = VALUES["complex"] + self.data = [Row(self.fields, vals) for vals in self.values] + self.layout_to_dictionary = dict() + + def create_dictionaries(self, source_): + for layout in LAYOUTS_COMPLEX: + if source_.compatible_with_layout(Layout(layout)): + self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields) + + def prepare(self, cluster_): + for _, dictionary in self.layout_to_dictionary.items(): + dictionary.prepare_source(cluster_) + dictionary.load_data(self.data) + + def execute(self, layout_name, node): + if not self.layout_to_dictionary.has_key(layout_name): + raise RuntimeError("Source doesn't support layout: {}".format(layout_name)) + + dct = self.layout_to_dictionary[layout_name] + + node.query("system reload dictionaries") + queries_with_answers = [] + + for row in self.data: + for field in self.fields: + if not field.is_key: + for query in dct.get_select_get_queries(field, row): + queries_with_answers.append((query, row.get_value_by_name(field.name))) + + for query in dct.get_select_has_queries(field, row): + queries_with_answers.append((query, 1)) + + for query in dct.get_select_get_or_default_queries(field, row): + queries_with_answers.append((query, field.default_value_for_get)) + + for query, answer in queries_with_answers: + # print query + assert node.query(query) == str(answer) + '\n' + + +class RangedLayoutTester: + def __init__(self): + self.fields = KEY_FIELDS["ranged"] + START_FIELDS["ranged"] + MIDDLE_FIELDS + END_FIELDS["ranged"] + self.values = VALUES["ranged"] + self.data = [Row(self.fields, vals) for vals in self.values] + self.layout_to_dictionary = dict() + + def create_dictionaries(self, source_): + for layout in LAYOUTS_RANGED: + if source_.compatible_with_layout(Layout(layout)): + self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields) + + def prepare(self, cluster_): + for _, dictionary in self.layout_to_dictionary.items(): + dictionary.prepare_source(cluster_) + dictionary.load_data(self.data) + + def execute(self, layout_name, node): + + if not self.layout_to_dictionary.has_key(layout_name): + raise RuntimeError("Source doesn't support layout: {}".format(layout_name)) + + dct = self.layout_to_dictionary[layout_name] + + node.query("system reload dictionaries") + + queries_with_answers = [] + for row in self.data: + for field in self.fields: + if not field.is_key and not field.is_range: + for query in dct.get_select_get_queries(field, row): + queries_with_answers.append((query, row.get_value_by_name(field.name))) + + for query, answer in queries_with_answers: + # print query + assert node.query(query) == str(answer) + '\n' + diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/configs/config.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/config.xml similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/configs/config.xml rename to tests/integration/test_dictionaries_all_layouts_separate_sources/configs/config.xml diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/configs/dictionaries/.gitkeep b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/dictionaries/.gitkeep similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/configs/dictionaries/.gitkeep rename to tests/integration/test_dictionaries_all_layouts_separate_sources/configs/dictionaries/.gitkeep diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/configs/disable_ssl_verification.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/disable_ssl_verification.xml similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/configs/disable_ssl_verification.xml rename to tests/integration/test_dictionaries_all_layouts_separate_sources/configs/disable_ssl_verification.xml diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/configs/enable_dictionaries.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/enable_dictionaries.xml similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/configs/enable_dictionaries.xml rename to tests/integration/test_dictionaries_all_layouts_separate_sources/configs/enable_dictionaries.xml diff --git a/tests/integration/test_dictionaries_all_layouts_and_sources/configs/users.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/users.xml similarity index 100% rename from tests/integration/test_dictionaries_all_layouts_and_sources/configs/users.xml rename to tests/integration/test_dictionaries_all_layouts_separate_sources/configs/users.xml diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py new file mode 100644 index 00000000000..c6b2ed370f4 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceCassandra + +SOURCE = SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py new file mode 100644 index 00000000000..c8597708c04 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceClickHouse + +SOURCE = SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py new file mode 100644 index 00000000000..4f190e4c332 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py @@ -0,0 +1,84 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceClickHouse + +SOURCE = SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + cluster.add_instance('clickhouse1', main_configs=main_configs) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("cache"))) ) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_COMPLEX).difference(set("complex_key_cache")))) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py new file mode 100644 index 00000000000..87b73d3acc1 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceExecutableCache + +SOURCE = SourceExecutableCache("ExecutableCache", "localhost", "9000", "node", "9000", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py new file mode 100644 index 00000000000..5ab5ee545e7 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceExecutableHashed + +SOURCE = SourceExecutableHashed("ExecutableHashed", "localhost", "9000", "node", "9000", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py new file mode 100644 index 00000000000..893d095a143 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceFile + +SOURCE = SourceFile("File", "localhost", "9000", "node", "9000", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("cache, direct"))) ) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("complex_key_cache, complex_key_direct")))) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py new file mode 100644 index 00000000000..8c7519fd0a7 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py @@ -0,0 +1,84 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceHTTP + +SOURCE = SourceHTTP("SourceHTTP", "localhost", "9000", "clickhouse1", "9000", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + cluster.add_instance('clickhouse1', main_configs=main_configs) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py new file mode 100644 index 00000000000..c5280aa0076 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py @@ -0,0 +1,84 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceHTTPS + +SOURCE = SourceHTTPS("SourceHTTPS", "localhost", "9000", "clickhouse1", "9000", "", "") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + cluster.add_instance('clickhouse1', main_configs=main_configs) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py new file mode 100644 index 00000000000..ffa376dcdb3 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceMongo + +SOURCE = SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py new file mode 100644 index 00000000000..53edfc4bbd2 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceMongoURI + +SOURCE = SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py new file mode 100644 index 00000000000..69e2543f226 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py @@ -0,0 +1,82 @@ +import os +import math +import pytest + +from .common import * + +from helpers.cluster import ClickHouseCluster +from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout +from helpers.external_sources import SourceMySQL + +SOURCE = SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse") + +cluster = None +node = None +simple_tester = None +complex_tester = None +ranged_tester = None + + +def setup_module(module): + global cluster + global node + global simple_tester + global complex_tester + global ranged_tester + + for f in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, f)) + + simple_tester = SimpleLayoutTester() + simple_tester.create_dictionaries(SOURCE) + + complex_tester = ComplexLayoutTester() + complex_tester.create_dictionaries(SOURCE) + + ranged_tester = RangedLayoutTester() + ranged_tester.create_dictionaries(SOURCE) + # Since that all .xml configs were created + + cluster = ClickHouseCluster(__file__) + + dictionaries = [] + main_configs = [] + main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + + for fname in os.listdir(DICT_CONFIG_PATH): + dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) + + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True) + + +def teardown_module(module): + global DICT_CONFIG_PATH + for fname in os.listdir(DICT_CONFIG_PATH): + os.remove(os.path.join(DICT_CONFIG_PATH, fname)) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + simple_tester.prepare(cluster) + complex_tester.prepare(cluster) + ranged_tester.prepare(cluster) + + yield cluster + + finally: + cluster.shutdown() + +@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +def test_simple(started_cluster, layout_name): + simple_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +def test_complex(started_cluster, layout_name): + complex_tester.execute(layout_name, node) + +@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) +def test_ranged(started_cluster, layout_name): + ranged_tester.execute(layout_name, node) From 91cbd49e85ee3de6c93719e17bbf7e7e422d889e Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 22 Sep 2020 00:09:50 +0300 Subject: [PATCH 247/390] fix tests --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +- .../System/StorageSystemZooKeeper.cpp | 20 +-- .../test.py | 23 ++-- .../integration/test_distributed_ddl/test.py | 12 +- .../configs/users.d/not_optimize_count.xml | 1 - .../test_polymorphic_parts/test.py | 45 +++--- .../configs/config.d/remote_servers.xml | 20 +++ tests/integration/test_quorum_inserts/test.py | 66 +++++---- .../configs/config.d/storage_conf.xml | 24 ++++ .../test_replicated_merge_tree_s3/test.py | 13 +- tests/integration/test_row_policy/test.py | 4 +- .../01038_dictionary_lifetime_min_zero_sec.sh | 28 ++-- .../01268_dictionary_direct_layout.sql | 128 +++++++++--------- tests/queries/skip_list.json | 13 -- 14 files changed, 216 insertions(+), 185 deletions(-) create mode 100644 tests/integration/test_quorum_inserts/configs/config.d/remote_servers.xml diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d7230940bb2..cc39d6fc8a3 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -858,11 +858,9 @@ BlockIO InterpreterCreateQuery::execute() auto & create = query_ptr->as(); if (!create.cluster.empty()) { - /// Allows to execute ON CLUSTER queries during version upgrade - bool force_backward_compatibility = !context.getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil; /// For CREATE query generate UUID on initiator, so it will be the same on all hosts. /// It will be ignored if database does not support UUIDs. - if (!force_backward_compatibility && !create.attach && create.uuid == UUIDHelpers::Nil) + if (!create.attach && create.uuid == UUIDHelpers::Nil) create.uuid = UUIDHelpers::generateV4(); return executeDDLQueryOnCluster(query_ptr, context, getRequiredAccess()); } diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index 17ab4ed4efb..81a42f1fe63 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -42,7 +43,7 @@ NamesAndTypesList StorageSystemZooKeeper::getNamesAndTypes() } -static bool extractPathImpl(const IAST & elem, String & res) +static bool extractPathImpl(const IAST & elem, String & res, const Context & context) { const auto * function = elem.as(); if (!function) @@ -51,7 +52,7 @@ static bool extractPathImpl(const IAST & elem, String & res) if (function->name == "and") { for (const auto & child : function->arguments->children) - if (extractPathImpl(*child, res)) + if (extractPathImpl(*child, res, context)) return true; return false; @@ -60,23 +61,24 @@ static bool extractPathImpl(const IAST & elem, String & res) if (function->name == "equals") { const auto & args = function->arguments->as(); - const IAST * value; + ASTPtr value; if (args.children.size() != 2) return false; const ASTIdentifier * ident; if ((ident = args.children.at(0)->as())) - value = args.children.at(1).get(); + value = args.children.at(1); else if ((ident = args.children.at(1)->as())) - value = args.children.at(0).get(); + value = args.children.at(0); else return false; if (ident->name != "path") return false; - const auto * literal = value->as(); + auto evaluated = evaluateConstantExpressionAsLiteral(value, context); + const auto * literal = evaluated->as(); if (!literal) return false; @@ -93,20 +95,20 @@ static bool extractPathImpl(const IAST & elem, String & res) /** Retrieve from the query a condition of the form `path = 'path'`, from conjunctions in the WHERE clause. */ -static String extractPath(const ASTPtr & query) +static String extractPath(const ASTPtr & query, const Context & context) { const auto & select = query->as(); if (!select.where()) return ""; String res; - return extractPathImpl(*select.where(), res) ? res : ""; + return extractPathImpl(*select.where(), res, context) ? res : ""; } void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const { - String path = extractPath(query_info.query); + String path = extractPath(query_info.query, context); if (path.empty()) throw Exception("SELECT from system.zookeeper table must contain condition like path = 'path' in WHERE clause.", ErrorCodes::BAD_ARGUMENTS); diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index df8bd6ab56f..45404c1170c 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -17,6 +17,7 @@ node4 = cluster.add_instance('node4') def started_cluster(): try: cluster.start() + yield cluster finally: cluster.shutdown() @@ -141,22 +142,24 @@ def test_backup_from_old_version_config(started_cluster): def test_backup_and_alter(started_cluster): - node4.query("CREATE TABLE backup_table(A Int64, B String, C Date) Engine = MergeTree order by tuple()") + node4.query("CREATE DATABASE test ENGINE=Ordinary") - node4.query("INSERT INTO backup_table VALUES(2, '2', toDate('2019-10-01'))") + node4.query("CREATE TABLE test.backup_table(A Int64, B String, C Date) Engine = MergeTree order by tuple()") - node4.query("ALTER TABLE backup_table FREEZE PARTITION tuple();") + node4.query("INSERT INTO test.backup_table VALUES(2, '2', toDate('2019-10-01'))") - node4.query("ALTER TABLE backup_table DROP COLUMN C") + node4.query("ALTER TABLE test.backup_table FREEZE PARTITION tuple();") - node4.query("ALTER TABLE backup_table MODIFY COLUMN B UInt64") + node4.query("ALTER TABLE test.backup_table DROP COLUMN C") - node4.query("ALTER TABLE backup_table DROP PARTITION tuple()") + node4.query("ALTER TABLE test.backup_table MODIFY COLUMN B UInt64") + + node4.query("ALTER TABLE test.backup_table DROP PARTITION tuple()") node4.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached']) + 'cp -r /var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/ /var/lib/clickhouse/data/test/backup_table/detached']) - node4.query("ALTER TABLE backup_table ATTACH PARTITION tuple()") + node4.query("ALTER TABLE test.backup_table ATTACH PARTITION tuple()") - assert node4.query("SELECT sum(A) FROM backup_table") == "2\n" - assert node4.query("SELECT B + 2 FROM backup_table") == "4\n" + assert node4.query("SELECT sum(A) FROM test.backup_table") == "2\n" + assert node4.query("SELECT B + 2 FROM test.backup_table") == "4\n" diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 08027fa13ca..7f254d03015 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -326,19 +326,15 @@ def test_socket_timeout(test_cluster): def test_replicated_without_arguments(test_cluster): rules = test_cluster.pm_random_drops.pop_rules() instance = test_cluster.instances['ch1'] - test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic", - settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) + test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic") test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", - settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster") test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", - settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", - settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.pm_random_drops.push_rules(rules) diff --git a/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml b/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml index 82689093adf..5a06453b214 100644 --- a/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml +++ b/tests/integration/test_polymorphic_parts/configs/users.d/not_optimize_count.xml @@ -2,7 +2,6 @@ 0 - Ordinary diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index dbbf5c0b4ff..39745e4f3f0 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -336,7 +336,7 @@ def test_polymorphic_parts_non_adaptive(start_cluster): "Wide\t2\n") assert node1.contains_in_log( - " default.non_adaptive_table: Table can't create parts with adaptive granularity") + " default.non_adaptive_table ([0-9a-f-]*): Table can't create parts with adaptive granularity") def test_in_memory(start_cluster): @@ -408,24 +408,29 @@ def test_in_memory_wal(start_cluster): pm.partition_instances(node11, node12) check(node11, 300, 6) - wal_file = os.path.join(node11.path, "database/data/default/wal_table/wal.bin") + wal_file = "/var/lib/clickhouse/data/default/wal_table/wal.bin" # Corrupt wal file - open(wal_file, 'rw+').truncate(os.path.getsize(wal_file) - 10) + # Truncate it to it's size minus 10 bytes + node11.exec_in_container(['bash', '-c', 'truncate --size="$(($(stat -c "%s" {}) - 10))" {}'.format(wal_file, wal_file)], + privileged=True, user='root') node11.restart_clickhouse(kill=True) # Broken part is lost, but other restored successfully check(node11, 250, 5) # WAL with blocks from 0 to 4 - broken_wal_file = os.path.join(node11.path, "database/data/default/wal_table/wal_0_4.bin") - assert os.path.exists(broken_wal_file) + broken_wal_file = "/var/lib/clickhouse/data/default/wal_table/wal_0_4.bin" + # Check file exists + node11.exec_in_container(['bash', '-c', 'test -f {}'.format(broken_wal_file)]) # Fetch lost part from replica node11.query("SYSTEM SYNC REPLICA wal_table", timeout=20) check(node11, 300, 6) # Check that new data is written to new wal, but old is still exists for restoring - assert os.path.getsize(wal_file) > 0 - assert os.path.exists(broken_wal_file) + # Check file not empty + node11.exec_in_container(['bash', '-c', 'test -s {}'.format(wal_file)]) + # Check file exists + node11.exec_in_container(['bash', '-c', 'test -f {}'.format(broken_wal_file)]) # Data is lost without WAL node11.query("ALTER TABLE wal_table MODIFY SETTING in_memory_parts_enable_wal = 0") @@ -446,8 +451,8 @@ def test_in_memory_wal_rotate(start_cluster): insert_random_data('restore_table', node11, 50) for i in range(5): - wal_file = os.path.join(node11.path, "database/data/default/restore_table/wal_{0}_{0}.bin".format(i)) - assert os.path.exists(wal_file) + # Check file exists + node11.exec_in_container(['bash', '-c', 'test -f /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin'.format(i)]) for node in [node11, node12]: node.query( @@ -459,13 +464,14 @@ def test_in_memory_wal_rotate(start_cluster): node11.restart_clickhouse(kill=True) for i in range(5): - wal_file = os.path.join(node11.path, "database/data/default/restore_table/wal_{0}_{0}.bin".format(i)) - assert not os.path.exists(wal_file) + # check file doesn't exist + node11.exec_in_container(['bash', '-c', 'test ! -e /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin'.format(i)]) # New wal file was created and ready to write part to it - wal_file = os.path.join(node11.path, "database/data/default/restore_table/wal.bin") - assert os.path.exists(wal_file) - assert os.path.getsize(wal_file) == 0 + # Check file exists + node11.exec_in_container(['bash', '-c', 'test -f /var/lib/clickhouse/data/default/restore_table/wal.bin']) + # Chech file empty + node11.exec_in_container(['bash', '-c', 'test ! -s /var/lib/clickhouse/data/default/restore_table/wal.bin']) def test_in_memory_deduplication(start_cluster): @@ -509,19 +515,20 @@ def test_in_memory_alters(start_cluster): def test_polymorphic_parts_index(start_cluster): + node1.query('CREATE DATABASE test_index ENGINE=Ordinary') node1.query(''' - CREATE TABLE index_compact(a UInt32, s String) + CREATE TABLE test_index.index_compact(a UInt32, s String) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100''') - node1.query("INSERT INTO index_compact SELECT number, toString(number) FROM numbers(100)") - node1.query("INSERT INTO index_compact SELECT number, toString(number) FROM numbers(30)") - node1.query("OPTIMIZE TABLE index_compact FINAL") + node1.query("INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(100)") + node1.query("INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(30)") + node1.query("OPTIMIZE TABLE test_index.index_compact FINAL") assert node1.query("SELECT part_type FROM system.parts WHERE table = 'index_compact' AND active") == "Compact\n" assert node1.query("SELECT marks FROM system.parts WHERE table = 'index_compact' AND active") == "2\n" - index_path = os.path.join(node1.path, "database/data/default/index_compact/all_1_2_1/primary.idx") + index_path = os.path.join(node1.path, "database/data/test_index/index_compact/all_1_2_1/primary.idx") f = open(index_path, 'rb') assert os.path.getsize(index_path) == 8 diff --git a/tests/integration/test_quorum_inserts/configs/config.d/remote_servers.xml b/tests/integration/test_quorum_inserts/configs/config.d/remote_servers.xml new file mode 100644 index 00000000000..b1cd417f8b9 --- /dev/null +++ b/tests/integration/test_quorum_inserts/configs/config.d/remote_servers.xml @@ -0,0 +1,20 @@ + + + + + + zero + 9000 + + + first + 9000 + + + second + 9000 + + + + + diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 0adee0afc64..2211333bb26 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -7,23 +7,21 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) zero = cluster.add_instance("zero", user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], macros={"cluster": "anime", "shard": "0", "replica": "zero"}, with_zookeeper=True) first = cluster.add_instance("first", user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], macros={"cluster": "anime", "shard": "0", "replica": "first"}, with_zookeeper=True) second = cluster.add_instance("second", user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], macros={"cluster": "anime", "shard": "0", "replica": "second"}, with_zookeeper=True) -def execute_on_all_cluster(query_): - for node in [zero, first, second]: - node.query(query_) - - @pytest.fixture(scope="module") def started_cluster(): global cluster @@ -36,7 +34,7 @@ def started_cluster(): def test_simple_add_replica(started_cluster): - execute_on_all_cluster("DROP TABLE IF EXISTS test_simple") + zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") create_query = "CREATE TABLE test_simple " \ "(a Int8, d Date) " \ @@ -67,11 +65,11 @@ def test_simple_add_replica(started_cluster): assert '1\t2011-01-01\n' == first.query("SELECT * from test_simple") assert '1\t2011-01-01\n' == second.query("SELECT * from test_simple") - execute_on_all_cluster("DROP TABLE IF EXISTS test_simple") + zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") def test_drop_replica_and_achieve_quorum(started_cluster): - execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") + zero.query("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster") create_query = "CREATE TABLE test_drop_replica_and_achieve_quorum " \ "(a Int8, d Date) " \ @@ -125,7 +123,7 @@ def test_drop_replica_and_achieve_quorum(started_cluster): assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) - execute_on_all_cluster("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum") + zero.query("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster") @pytest.mark.parametrize( @@ -136,17 +134,15 @@ def test_drop_replica_and_achieve_quorum(started_cluster): ] ) def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): - execute_on_all_cluster("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition") + zero.query("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster") - create_query = "CREATE TABLE test_quorum_insert_with_drop_partition " \ + create_query = "CREATE TABLE test_quorum_insert_with_drop_partition ON CLUSTER cluster " \ "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "Engine = ReplicatedMergeTree " \ "PARTITION BY d ORDER BY a " print("Create Replicated table with three replicas") zero.query(create_query) - first.query(create_query) - second.query(create_query) print("Stop fetches for test_quorum_insert_with_drop_partition at first replica.") first.query("SYSTEM STOP FETCHES test_quorum_insert_with_drop_partition") @@ -167,9 +163,11 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): print("Sync first replica with others.") first.query("SYSTEM SYNC REPLICA test_quorum_insert_with_drop_partition") - assert "20110101" not in first.query("SELECT * FROM system.zookeeper " \ - "where path='/clickhouse/tables/0/test_quorum_insert_with_drop_partition/quorum/last_part' " \ - "format Vertical") + assert "20110101" not in first.query(""" + WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_quorum_insert_with_drop_partition') AS uuid, + '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p + SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical + """) print("Select from updated partition.") if (add_new_data): @@ -179,7 +177,7 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): assert TSV("") == TSV(zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")) assert TSV("") == TSV(second.query("SELECT * FROM test_quorum_insert_with_drop_partition")) - execute_on_all_cluster("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition") + zero.query("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster") @pytest.mark.parametrize( @@ -190,28 +188,24 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): ] ) def test_insert_quorum_with_move_partition(started_cluster, add_new_data): - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source") - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster") - create_source = "CREATE TABLE test_insert_quorum_with_move_partition_source " \ + create_source = "CREATE TABLE test_insert_quorum_with_move_partition_source ON CLUSTER cluster " \ "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "Engine = ReplicatedMergeTree " \ "PARTITION BY d ORDER BY a " - create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination " \ + create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination ON CLUSTER cluster " \ "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ + "Engine = ReplicatedMergeTree " \ "PARTITION BY d ORDER BY a " print("Create source Replicated table with three replicas") zero.query(create_source) - first.query(create_source) - second.query(create_source) print("Create destination Replicated table with three replicas") zero.query(create_destination) - first.query(create_destination) - second.query(create_destination) print("Stop fetches for test_insert_quorum_with_move_partition_source at first replica.") first.query("SYSTEM STOP FETCHES test_insert_quorum_with_move_partition_source") @@ -233,9 +227,11 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): print("Sync first replica with others.") first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_move_partition_source") - assert "20110101" not in first.query("SELECT * FROM system.zookeeper " \ - "where path='/clickhouse/tables/0/test_insert_quorum_with_move_partition_source/quorum/last_part' " \ - "format Vertical") + assert "20110101" not in first.query(""" + WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_insert_quorum_with_move_partition_source') AS uuid, + '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p + SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical + """) print("Select from updated partition.") if (add_new_data): @@ -246,12 +242,12 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): assert TSV("") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) assert TSV("") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source") - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster") def test_insert_quorum_with_ttl(started_cluster): - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") create_query = "CREATE TABLE test_insert_quorum_with_ttl " \ "(a Int8, d Date) " \ @@ -298,4 +294,4 @@ def test_insert_quorum_with_ttl(started_cluster): assert TSV("2\t2012-02-02\n") == TSV( first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 1})) - execute_on_all_cluster("DROP TABLE IF EXISTS test_insert_quorum_with_ttl") + zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml index f3b7f959ce9..20b750ffff3 100644 --- a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -22,4 +22,28 @@ 0 + + + + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + + + + + 0 + + diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 1414905759a..4d19793d0b2 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -14,11 +14,11 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '1'}, with_minio=True, with_zookeeper=True) - cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '2'}, with_zookeeper=True) - cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'cluster': 'test1'}, + cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '3'}, with_zookeeper=True) logging.info("Starting cluster...") @@ -49,12 +49,12 @@ def generate_values(date_str, count, sign=1): def create_table(cluster, additional_settings=None): create_table_statement = """ - CREATE TABLE s3_test ( + CREATE TABLE s3_test ON CLUSTER cluster( dt Date, id Int64, data String, INDEX min_max (id) TYPE minmax GRANULARITY 3 - ) ENGINE=ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/s3', '{instance}') + ) ENGINE=ReplicatedMergeTree() PARTITION BY dt ORDER BY (dt, id) SETTINGS storage_policy='s3' @@ -63,8 +63,7 @@ def create_table(cluster, additional_settings=None): create_table_statement += "," create_table_statement += additional_settings - for node in cluster.instances.values(): - node.query(create_table_statement) + cluster.instances.values()[0].query(create_table_statement) @pytest.fixture(autouse=True) diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index a407f0b2c7a..c3c86f5a9c5 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -34,7 +34,7 @@ def started_cluster(): for current_node in nodes: current_node.query(''' - CREATE DATABASE mydb ENGINE=Ordinary; + CREATE DATABASE mydb; CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); @@ -360,7 +360,7 @@ def test_miscellaneous_engines(): # ReplicatedCollapsingMergeTree node.query("DROP TABLE mydb.filtered_table1") node.query( - "CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") + "CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-01/filtered_table1', 'replica1', b) ORDER BY a") node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) diff --git a/tests/queries/0_stateless/01038_dictionary_lifetime_min_zero_sec.sh b/tests/queries/0_stateless/01038_dictionary_lifetime_min_zero_sec.sh index c3643399ba1..48171b56dd3 100755 --- a/tests/queries/0_stateless/01038_dictionary_lifetime_min_zero_sec.sh +++ b/tests/queries/0_stateless/01038_dictionary_lifetime_min_zero_sec.sh @@ -3,13 +3,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS db_01038" -$CLICKHOUSE_CLIENT --query "CREATE DATABASE database_for_dict Engine = Ordinary" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE db_01038" $CLICKHOUSE_CLIENT --query " -CREATE TABLE database_for_dict.table_for_dict +CREATE TABLE db_01038.table_for_dict ( key_column UInt64, value Float64 @@ -17,34 +17,34 @@ CREATE TABLE database_for_dict.table_for_dict ENGINE = MergeTree() ORDER BY key_column" -$CLICKHOUSE_CLIENT --query "INSERT INTO database_for_dict.table_for_dict VALUES (1, 1.1)" +$CLICKHOUSE_CLIENT --query "INSERT INTO db_01038.table_for_dict VALUES (1, 1.1)" $CLICKHOUSE_CLIENT --query " -CREATE DICTIONARY database_for_dict.dict_with_zero_min_lifetime +CREATE DICTIONARY db_01038.dict_with_zero_min_lifetime ( key_column UInt64, value Float64 DEFAULT 77.77 ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' DB 'db_01038')) LIFETIME(1) LAYOUT(FLAT())" -$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(1))" +$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(1))" -$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(2))" +$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(2))" -$CLICKHOUSE_CLIENT --query "INSERT INTO database_for_dict.table_for_dict VALUES (2, 2.2)" +$CLICKHOUSE_CLIENT --query "INSERT INTO db_01038.table_for_dict VALUES (2, 2.2)" function check() { - query_result=$($CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(2))") + query_result=$($CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(2))") while [ "$query_result" != "2.2" ] do - query_result=$($CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(2))") + query_result=$($CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(2))") done } @@ -53,8 +53,8 @@ export -f check; timeout 10 bash -c check -$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(1))" +$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(1))" -$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('database_for_dict.dict_with_zero_min_lifetime', 'value', toUInt64(2))" +$CLICKHOUSE_CLIENT --query "SELECT dictGetFloat64('db_01038.dict_with_zero_min_lifetime', 'value', toUInt64(2))" -$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS db_01038" diff --git a/tests/queries/0_stateless/01268_dictionary_direct_layout.sql b/tests/queries/0_stateless/01268_dictionary_direct_layout.sql index 9b2f2344242..48642c91102 100644 --- a/tests/queries/0_stateless/01268_dictionary_direct_layout.sql +++ b/tests/queries/0_stateless/01268_dictionary_direct_layout.sql @@ -1,12 +1,12 @@ -DROP DATABASE IF EXISTS database_for_dict; +DROP DATABASE IF EXISTS database_for_dict_01268; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict_01268; -DROP TABLE IF EXISTS database_for_dict.table_for_dict1; -DROP TABLE IF EXISTS database_for_dict.table_for_dict2; -DROP TABLE IF EXISTS database_for_dict.table_for_dict3; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict1; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict2; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict3; -CREATE TABLE database_for_dict.table_for_dict1 +CREATE TABLE database_for_dict_01268.table_for_dict1 ( key_column UInt64, second_column UInt64, @@ -15,9 +15,9 @@ CREATE TABLE database_for_dict.table_for_dict1 ENGINE = MergeTree() ORDER BY key_column; -INSERT INTO database_for_dict.table_for_dict1 VALUES (100500, 10000000, 'Hello world'); +INSERT INTO database_for_dict_01268.table_for_dict1 VALUES (100500, 10000000, 'Hello world'); -CREATE TABLE database_for_dict.table_for_dict2 +CREATE TABLE database_for_dict_01268.table_for_dict2 ( region_id UInt64, parent_region UInt64, @@ -26,13 +26,13 @@ CREATE TABLE database_for_dict.table_for_dict2 ENGINE = MergeTree() ORDER BY region_id; -INSERT INTO database_for_dict.table_for_dict2 VALUES (1, 0, 'Russia'); -INSERT INTO database_for_dict.table_for_dict2 VALUES (2, 1, 'Moscow'); -INSERT INTO database_for_dict.table_for_dict2 VALUES (3, 2, 'Center'); -INSERT INTO database_for_dict.table_for_dict2 VALUES (4, 0, 'Great Britain'); -INSERT INTO database_for_dict.table_for_dict2 VALUES (5, 4, 'London'); +INSERT INTO database_for_dict_01268.table_for_dict2 VALUES (1, 0, 'Russia'); +INSERT INTO database_for_dict_01268.table_for_dict2 VALUES (2, 1, 'Moscow'); +INSERT INTO database_for_dict_01268.table_for_dict2 VALUES (3, 2, 'Center'); +INSERT INTO database_for_dict_01268.table_for_dict2 VALUES (4, 0, 'Great Britain'); +INSERT INTO database_for_dict_01268.table_for_dict2 VALUES (5, 4, 'London'); -CREATE TABLE database_for_dict.table_for_dict3 +CREATE TABLE database_for_dict_01268.table_for_dict3 ( region_id UInt64, parent_region Float32, @@ -41,91 +41,91 @@ CREATE TABLE database_for_dict.table_for_dict3 ENGINE = MergeTree() ORDER BY region_id; -INSERT INTO database_for_dict.table_for_dict3 VALUES (1, 0.5, 'Russia'); -INSERT INTO database_for_dict.table_for_dict3 VALUES (2, 1.6, 'Moscow'); -INSERT INTO database_for_dict.table_for_dict3 VALUES (3, 2.3, 'Center'); -INSERT INTO database_for_dict.table_for_dict3 VALUES (4, 0.2, 'Great Britain'); -INSERT INTO database_for_dict.table_for_dict3 VALUES (5, 4.9, 'London'); +INSERT INTO database_for_dict_01268.table_for_dict3 VALUES (1, 0.5, 'Russia'); +INSERT INTO database_for_dict_01268.table_for_dict3 VALUES (2, 1.6, 'Moscow'); +INSERT INTO database_for_dict_01268.table_for_dict3 VALUES (3, 2.3, 'Center'); +INSERT INTO database_for_dict_01268.table_for_dict3 VALUES (4, 0.2, 'Great Britain'); +INSERT INTO database_for_dict_01268.table_for_dict3 VALUES (5, 4.9, 'London'); -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01268; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE db_01268; -DROP DICTIONARY IF EXISTS ordinary_db.dict1; -DROP DICTIONARY IF EXISTS ordinary_db.dict2; -DROP DICTIONARY IF EXISTS ordinary_db.dict3; +DROP DICTIONARY IF EXISTS db_01268.dict1; +DROP DICTIONARY IF EXISTS db_01268.dict2; +DROP DICTIONARY IF EXISTS db_01268.dict3; -CREATE DICTIONARY ordinary_db.dict1 +CREATE DICTIONARY db_01268.dict1 ( key_column UInt64 DEFAULT 0, second_column UInt64 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB 'database_for_dict_01268')) LAYOUT(DIRECT()) SETTINGS(max_result_bytes=1); -CREATE DICTIONARY ordinary_db.dict2 +CREATE DICTIONARY db_01268.dict2 ( region_id UInt64 DEFAULT 0, parent_region UInt64 DEFAULT 0 HIERARCHICAL, region_name String DEFAULT '' ) PRIMARY KEY region_id -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB 'database_for_dict_01268')) LAYOUT(DIRECT()); -CREATE DICTIONARY ordinary_db.dict3 +CREATE DICTIONARY db_01268.dict3 ( region_id UInt64 DEFAULT 0, parent_region Float32 DEFAULT 0, region_name String DEFAULT '' ) PRIMARY KEY region_id -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict3' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict3' PASSWORD '' DB 'database_for_dict_01268')) LAYOUT(DIRECT()); SELECT 'INITIALIZING DICTIONARY'; -SELECT dictGetHierarchy('ordinary_db.dict2', toUInt64(3)); -SELECT dictHas('ordinary_db.dict2', toUInt64(3)); -SELECT dictHas('ordinary_db.dict2', toUInt64(45)); -SELECT dictIsIn('ordinary_db.dict2', toUInt64(3), toUInt64(1)); -SELECT dictIsIn('ordinary_db.dict2', toUInt64(1), toUInt64(3)); -SELECT dictGetUInt64('ordinary_db.dict2', 'parent_region', toUInt64(3)); -SELECT dictGetUInt64('ordinary_db.dict2', 'parent_region', toUInt64(99)); -SELECT dictGetFloat32('ordinary_db.dict3', 'parent_region', toUInt64(3)); -SELECT dictGetFloat32('ordinary_db.dict3', 'parent_region', toUInt64(2)); -SELECT dictGetFloat32('ordinary_db.dict3', 'parent_region', toUInt64(1)); -SELECT dictGetString('ordinary_db.dict2', 'region_name', toUInt64(5)); -SELECT dictGetString('ordinary_db.dict2', 'region_name', toUInt64(4)); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(100), 'NONE'); +SELECT dictGetHierarchy('db_01268.dict2', toUInt64(3)); +SELECT dictHas('db_01268.dict2', toUInt64(3)); +SELECT dictHas('db_01268.dict2', toUInt64(45)); +SELECT dictIsIn('db_01268.dict2', toUInt64(3), toUInt64(1)); +SELECT dictIsIn('db_01268.dict2', toUInt64(1), toUInt64(3)); +SELECT dictGetUInt64('db_01268.dict2', 'parent_region', toUInt64(3)); +SELECT dictGetUInt64('db_01268.dict2', 'parent_region', toUInt64(99)); +SELECT dictGetFloat32('db_01268.dict3', 'parent_region', toUInt64(3)); +SELECT dictGetFloat32('db_01268.dict3', 'parent_region', toUInt64(2)); +SELECT dictGetFloat32('db_01268.dict3', 'parent_region', toUInt64(1)); +SELECT dictGetString('db_01268.dict2', 'region_name', toUInt64(5)); +SELECT dictGetString('db_01268.dict2', 'region_name', toUInt64(4)); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(100), 'NONE'); -SELECT number + 1, dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(number + 1), 'NONE') chars FROM numbers(10); -SELECT number + 1, dictGetFloat32OrDefault('ordinary_db.dict3', 'parent_region', toUInt64(number + 1), toFloat32(0)) chars FROM numbers(10); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(1), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(2), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(3), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(4), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(5), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(6), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(7), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(8), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(9), 'NONE'); -SELECT dictGetStringOrDefault('ordinary_db.dict2', 'region_name', toUInt64(10), 'NONE'); +SELECT number + 1, dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(number + 1), 'NONE') chars FROM numbers(10); +SELECT number + 1, dictGetFloat32OrDefault('db_01268.dict3', 'parent_region', toUInt64(number + 1), toFloat32(0)) chars FROM numbers(10); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(1), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(2), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(3), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(4), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(5), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(6), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(7), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(8), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(9), 'NONE'); +SELECT dictGetStringOrDefault('db_01268.dict2', 'region_name', toUInt64(10), 'NONE'); -SELECT dictGetUInt64('ordinary_db.dict1', 'second_column', toUInt64(100500)); -- { serverError 396 } +SELECT dictGetUInt64('db_01268.dict1', 'second_column', toUInt64(100500)); -- { serverError 396 } SELECT 'END'; -DROP DICTIONARY IF EXISTS ordinary_db.dict1; -DROP DICTIONARY IF EXISTS ordinary_db.dict2; -DROP DICTIONARY IF EXISTS ordinary_db.dict3; +DROP DICTIONARY IF EXISTS db_01268.dict1; +DROP DICTIONARY IF EXISTS db_01268.dict2; +DROP DICTIONARY IF EXISTS db_01268.dict3; -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01268; -DROP TABLE IF EXISTS database_for_dict.table_for_dict1; -DROP TABLE IF EXISTS database_for_dict.table_for_dict2; -DROP TABLE IF EXISTS database_for_dict.table_for_dict3; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict1; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict2; +DROP TABLE IF EXISTS database_for_dict_01268.table_for_dict3; -DROP DATABASE IF EXISTS database_for_dict; +DROP DATABASE IF EXISTS database_for_dict_01268; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 4ffa2bd74cb..ea256cabdc4 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -89,18 +89,5 @@ "release-build": [ ], "database-atomic": [ - /// Inner tables of materialized views have different names - ///"00738_lock_for_inner_table", - ///"00609_mv_index_in_in", - ///"00510_materizlized_view_and_deduplication_zookeeper", - /// Different database engine - ///"00604_show_create_database", - /// UUID must be specified in ATTACH TABLE - ///"01190_full_attach_syntax", - /// Assumes blocking DROP - ///"01320_create_sync_race_condition", - /// Internal distionary name is different - ///"01225_show_create_table_from_dictionary", - ///"01224_no_superfluous_dict_reload" ] } From 40ff1cf9127eca5c6ab8b6bfabd689f28fea5fd3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 00:45:53 +0300 Subject: [PATCH 248/390] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 596fe20be90..6d57dfde9cd 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -38,6 +38,7 @@ toc_title: Adopters | Deutsche Bank | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | Diva-e | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | | Ecwid | E-commerce SaaS | Metrics, Logging | — | — | [Slides in Russian, April 2019](https://nastachku.ru/var/files/1/presentation/backend/2_Backend_6.pdf) | +| eBay | E-commerce | TBA | — | — | [Webinar, Sep 2020](https://altinity.com/webinarspage/2020/09/08/migrating-from-druid-to-next-gen-olap-on-clickhouse-ebays-experience) | | Exness | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | FastNetMon | DDoS Protection | Main Product | | — | [Official website](https://fastnetmon.com/docs-fnm-advanced/fastnetmon-advanced-traffic-persistency/) | | Flipkart | e-Commerce | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=239) | From 3c1a679612185e21a73847fd72ba6361496858e6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 02:06:19 +0300 Subject: [PATCH 249/390] Update performance-test.md --- docs/en/operations/performance-test.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md index c3ef10da774..734e63b0b35 100644 --- a/docs/en/operations/performance-test.md +++ b/docs/en/operations/performance-test.md @@ -13,9 +13,9 @@ With this instruction you can run basic ClickHouse performance test on any serve 4. ssh to the server and download it with wget: ```bash # For amd64: -wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse +wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse # For aarch64: -wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578161264_binary/clickhouse +wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse # Then do: chmod a+x clickhouse ``` From df122b90b4dab10d078a41032ed1c3c09513e15b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 02:07:12 +0300 Subject: [PATCH 250/390] Update performance-test.md --- docs/en/operations/performance-test.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md index 734e63b0b35..984bbe02174 100644 --- a/docs/en/operations/performance-test.md +++ b/docs/en/operations/performance-test.md @@ -19,43 +19,35 @@ wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e # Then do: chmod a+x clickhouse ``` -5. Download configs: -```bash -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml -mkdir config.d -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml -``` -6. Download benchmark files: +5. Download benchmark files: ```bash wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/benchmark-new.sh chmod a+x benchmark-new.sh wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql ``` -7. Download test data according to the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) instruction (“hits” table containing 100 million rows). +6. Download test data according to the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) instruction (“hits” table containing 100 million rows). ```bash wget https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz tar xvf hits_100m_obfuscated_v1.tar.xz -C . mv hits_100m_obfuscated_v1/* . ``` -8. Run the server: +7. Run the server: ```bash ./clickhouse server ``` -9. Check the data: ssh to the server in another terminal +8. Check the data: ssh to the server in another terminal ```bash ./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" 100000000 ``` -10. Edit the benchmark-new.sh, change `clickhouse-client` to `./clickhouse client` and add `--max_memory_usage 100000000000` parameter. +9. Edit the benchmark-new.sh, change `clickhouse-client` to `./clickhouse client` and add `--max_memory_usage 100000000000` parameter. ```bash mcedit benchmark-new.sh ``` -11. Run the benchmark: +10. Run the benchmark: ```bash ./benchmark-new.sh hits_100m_obfuscated ``` -12. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com +11. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com All the results are published here: https://clickhouse.tech/benchmark/hardware/ From f9402c95fa4599d94d7240601f989005bb2e15fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 04:17:40 +0300 Subject: [PATCH 251/390] Add benchmark results from seo.do --- website/benchmark/hardware/index.html | 3 +- .../hardware/results/052_amd_epyc_7642.json | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 website/benchmark/hardware/results/052_amd_epyc_7642.json diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html index ce7fb602258..68bc4b29653 100644 --- a/website/benchmark/hardware/index.html +++ b/website/benchmark/hardware/index.html @@ -68,7 +68,8 @@ Results for Yandex Managed ClickHouse for "cold cache" are biased and should not Results for AWS Lightsail is from Vamsi Krishna B.
Results for Dell XPS laptop and Google Pixel phone is from Alexander Kuzmenkov.
Results for Android phones for "cold cache" are done without cache flushing, so they are not "cold" and cannot be compared.
-Results for Digital Ocean are from Zimin Aleksey. +Results for Digital Ocean are from Zimin Aleksey.
+Results for 2x EPYC 7642 w/ 512 GB RAM (192 Cores) + 12X 1TB SSD (RAID6) are from Yiğit Konur and Metehan Çetinkaya of seo.do.

diff --git a/website/benchmark/hardware/results/052_amd_epyc_7642.json b/website/benchmark/hardware/results/052_amd_epyc_7642.json new file mode 100644 index 00000000000..b60146d515f --- /dev/null +++ b/website/benchmark/hardware/results/052_amd_epyc_7642.json @@ -0,0 +1,56 @@ +[ + { + "system": "AMD EPYC 7642", + "system_full": "2x AMD EPYC 7642 / 512 GB RAM / 12x 1TB SSD (RAID 6)", + "time": "2020-09-21 00:00:00", + "kind": "server", + "result": + [ + [0.003, 0.003, 0.002], + [0.039, 0.041, 0.024], + [0.052, 0.029, 0.029], + [0.087, 0.031, 0.032], + [0.152, 0.106, 0.105], + [0.204, 0.128, 0.128], + [0.049, 0.028, 0.027], + [0.031, 0.024, 0.027], + [0.190, 0.130, 0.125], + [0.210, 0.142, 0.138], + [0.142, 0.091, 0.087], + [0.143, 0.101, 0.097], + [0.318, 0.170, 0.163], + [0.303, 0.193, 0.191], + [0.240, 0.175, 0.166], + [0.200, 0.166, 0.161], + [0.466, 0.364, 0.345], + [0.298, 0.244, 0.231], + [1.288, 0.901, 0.859], + [0.087, 0.031, 0.025], + [0.663, 0.201, 0.191], + [0.661, 0.213, 0.154], + [1.118, 0.599, 0.593], + [1.708, 0.392, 0.318], + [0.202, 0.065, 0.066], + [0.135, 0.061, 0.057], + [0.203, 0.066, 0.067], + [0.630, 0.296, 0.290], + [0.578, 0.281, 0.262], + [0.662, 0.670, 0.639], + [0.241, 0.153, 0.150], + [0.424, 0.235, 0.231], + [1.505, 1.090, 1.090], + [1.038, 0.818, 0.799], + [1.064, 0.856, 0.809], + [0.332, 0.297, 0.275], + [0.200, 0.169, 0.168], + [0.083, 0.070, 0.071], + [0.090, 0.059, 0.063], + [0.416, 0.419, 0.398], + [0.048, 0.032, 0.032], + [0.036, 0.027, 0.025], + [0.007, 0.007, 0.007] + ] + } +] + + From 7db544e241e131f62493c71bc73b2b33912d673c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 05:19:11 +0300 Subject: [PATCH 252/390] Added a script to perform hardware benchmark in a single command --- benchmark/hardware.sh | 102 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100755 benchmark/hardware.sh diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh new file mode 100755 index 00000000000..3fd230ec73c --- /dev/null +++ b/benchmark/hardware.sh @@ -0,0 +1,102 @@ +#!/bin/bash -e + +mkdir -p clickhouse-benchmark +pushd clickhouse-benchmark + +TABLE="hits_100m_obfuscated" +DATASET="${TABLE}_v1.tar.xz" +QUERIES_FILE="queries.sql" +TRIES=3 + +FASTER_DOWNLOAD=wget +if command -v axel >/dev/null; then + FASTER_DOWNLOAD=axel +fi + +if [[ ! -f clickhouse ]]; then + CPU=$(uname -m) + if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then + $FASTER_DOWNLOAD "https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" + elif [[ $CPU == aarch64 ]]; then + $FASTER_DOWNLOAD "https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" + else + echo "Unsupported CPU type: $CPU" + exit 1 + fi +fi + +chmod a+x clickhouse + +if [[ ! -f $QUERIES_FILE ]]; then + wget "https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/$QUERIES_FILE" +fi + +if [[ ! -d data ]]; then + if [[ ! -f $DATASET ]]; then + $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET" + fi + + tar --strip-components=1 --directory=. -x -v -f $DATASET +fi + +echo "Starting clickhouse-server" + +./clickhouse server > server.log 2>&1 & +PID=$! + +function finish { + kill $PID + wait +} +trap finish EXIT + +echo "Waiting for clickhouse-server to start" + +for i in {1..30}; do + sleep 1 + ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" && break + if [[ $i == 30 ]]; then exit 1; fi +done + +echo +echo "Will perform benchmark. Results:" +echo + +cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do + sync + echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null + + echo -n "[" + for i in $(seq 1 $TRIES); do + RES=$(./clickhouse client --max_memory_usage 100000000000 --time --format=Null --query="$query" 2>&1) + [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" + [[ "$i" != $TRIES ]] && echo -n ", " + done + echo "]," +done + + +echo +echo "Benchmark complete. System info:" + +echo '----version and build id--------' +./clickhouse local --query "SELECT version(), buildId()" +echo '----CPU-------------------------' +lscpu +echo '----Block Devices---------------' +lsblk +echo '----Disk Free and Total--------' +df -h . +echo '----Memory Free and Total-------' +free -h +echo '----Physical Memory Amount------' +cat /proc/meminfo | grep MemTotal +echo '----RAID Info-------------------' +cat /proc/mdstat +#echo '----PCI-------------------------' +#lspci +#echo '----All Hardware Info-----------' +#lshw +echo '--------------------------------' + +echo From ddfd5122414ca498ad0792ccf250beee4bcb02fe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 05:22:53 +0300 Subject: [PATCH 253/390] Adjustments --- benchmark/hardware.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 3fd230ec73c..71cd836f2d0 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -54,7 +54,7 @@ echo "Waiting for clickhouse-server to start" for i in {1..30}; do sleep 1 - ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" && break + ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" 2>/dev/null && break || echo '.' if [[ $i == 30 ]]; then exit 1; fi done @@ -78,8 +78,9 @@ done echo echo "Benchmark complete. System info:" +echo -echo '----version and build id--------' +echo '----Version and build id--------' ./clickhouse local --query "SELECT version(), buildId()" echo '----CPU-------------------------' lscpu From 923724c0f4946fb24691cae6f7fc3870e2773a81 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 06:06:08 +0300 Subject: [PATCH 254/390] Improvements --- benchmark/hardware.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 71cd836f2d0..dc75cc79626 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -11,6 +11,14 @@ TRIES=3 FASTER_DOWNLOAD=wget if command -v axel >/dev/null; then FASTER_DOWNLOAD=axel +else + echo "It's recommended to install 'axel' for faster downloads." +fi + +if command -v pixz >/dev/null; then + TAR_PARAMS='-Ipixz' +else + echo "It's recommended to install 'pixz' for faster decompression of the dataset." fi if [[ ! -f clickhouse ]]; then @@ -36,7 +44,7 @@ if [[ ! -d data ]]; then $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET" fi - tar --strip-components=1 --directory=. -x -v -f $DATASET + tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET fi echo "Starting clickhouse-server" From 2f09e36778a16e4a2527f17c8dbe7fe927ac8f4d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 06:13:28 +0300 Subject: [PATCH 255/390] Allow scale 1000 in benchmark --- benchmark/hardware.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index dc75cc79626..6d2834d7164 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -1,9 +1,12 @@ #!/bin/bash -e -mkdir -p clickhouse-benchmark -pushd clickhouse-benchmark +if [[ -n $1 ]]; then + SCALE=$1 +else + SCALE=100 +fi -TABLE="hits_100m_obfuscated" +TABLE="hits_${SCALE}m_obfuscated" DATASET="${TABLE}_v1.tar.xz" QUERIES_FILE="queries.sql" TRIES=3 @@ -21,6 +24,9 @@ else echo "It's recommended to install 'pixz' for faster decompression of the dataset." fi +mkdir -p clickhouse-benchmark-$SCALE +pushd clickhouse-benchmark-$SCALE + if [[ ! -f clickhouse ]]; then CPU=$(uname -m) if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then From 4cd7de175e9402c97cf686a85ab22c6ed69fc44e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Sep 2020 06:17:09 +0300 Subject: [PATCH 256/390] Adjustments --- benchmark/hardware.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 6d2834d7164..0ff71df19ae 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -11,6 +11,9 @@ DATASET="${TABLE}_v1.tar.xz" QUERIES_FILE="queries.sql" TRIES=3 +AMD64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" +AARCH64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" + FASTER_DOWNLOAD=wget if command -v axel >/dev/null; then FASTER_DOWNLOAD=axel @@ -30,9 +33,9 @@ pushd clickhouse-benchmark-$SCALE if [[ ! -f clickhouse ]]; then CPU=$(uname -m) if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then - $FASTER_DOWNLOAD "https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" + $FASTER_DOWNLOAD "$AMD64_BIN_URL" elif [[ $CPU == aarch64 ]]; then - $FASTER_DOWNLOAD "https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" + $FASTER_DOWNLOAD "$AARCH64_BIN_URL" else echo "Unsupported CPU type: $CPU" exit 1 From a5b45dabf20f7eed913f69f061f06790eddc9a1c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Sep 2020 10:14:12 +0300 Subject: [PATCH 257/390] Update version_date.tsv after release 20.9.2.20 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 3ec9ee11b95..75605968a37 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v20.9.2.20-stable 2020-09-22 v20.8.3.18-stable 2020-09-18 v20.8.2.3-stable 2020-09-08 v20.7.3.7-stable 2020-09-18 From 639466afdb94f24f0d43a8a168d78f62a1adf93c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 22 Sep 2020 10:45:08 +0300 Subject: [PATCH 258/390] fix docker images --- docker/test/stateful/run.sh | 2 +- docker/test/stateless_unbundled/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 34980508488..616c24395a7 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -27,7 +27,7 @@ ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickho ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then #FIXME USE_DATABASE_ORDINARY - ln -s /usr/share/clickhouse-test/config/database_ordinary_configd.xml /etc/clickhouse-server/config.d/ + ln -s /usr/share/clickhouse-test/config/database_ordinary_usersd.xml /etc/clickhouse-server/config.d/ fi function start() diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index 13ded2290fb..e0338448759 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -44,7 +44,7 @@ if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; th ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ fi if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then #FIXME USE_DATABASE_ORDINARY - ln -s /usr/share/clickhouse-test/config/database_ordinary_configd.xml /etc/clickhouse-server/config.d/ + ln -s /usr/share/clickhouse-test/config/database_ordinary_usersd.xml /etc/clickhouse-server/config.d/ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml From d7c7619cd84bea7b6bc8c63b98911e29446c8732 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 22 Sep 2020 13:13:57 +0300 Subject: [PATCH 259/390] add test for https://github.com/ClickHouse/ClickHouse/issues/14570 --- .../0_stateless/01505_log_distributed_deadlock.reference | 0 .../queries/0_stateless/01505_log_distributed_deadlock.sql | 7 +++++++ 2 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.reference create mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.sql diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.reference b/tests/queries/0_stateless/01505_log_distributed_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql new file mode 100644 index 00000000000..b3f294df327 --- /dev/null +++ b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql @@ -0,0 +1,7 @@ +create table t_local(a int) engine Log; + +create table t_dist (a int) engine Distributed(test_shard_localhost, 'default', 't_local', cityHash64(a)); + +set insert_distributed_sync = 1; + +insert into t_dist values (1); From 2280482530b2dbb6d20073b22fb57a65ccb9e032 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 22 Sep 2020 13:16:22 +0300 Subject: [PATCH 260/390] Keep LowCardinality in MergeJoin left key result type (#15088) --- src/Interpreters/MergeJoin.cpp | 4 +- src/Interpreters/join_common.cpp | 15 ++++- src/Interpreters/join_common.h | 3 +- ...erge_join_lc_and_nullable_in_key.reference | 29 +++++++++ ...1142_merge_join_lc_and_nullable_in_key.sql | 48 ++++++++++++++ .../01477_lc_in_merge_join_left_key.reference | 35 ++++++++++ .../01477_lc_in_merge_join_left_key.sql | 65 +++++++++++++++++++ 7 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference create mode 100644 tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql create mode 100644 tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference create mode 100644 tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 0154f8453b3..c9072ec3480 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -602,7 +602,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); - JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft()); + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft(), false); sortBlock(block, left_sort_description); @@ -636,6 +636,8 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) /// Back thread even with no data. We have some unfinished data in buffer. if (!not_processed && left_blocks_buffer) not_processed = std::make_shared(NotProcessed{{}, 0, 0, 0}); + + JoinCommon::restoreLowCardinalityInplace(block); } template diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 866893fa359..17c289b151d 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -185,13 +185,24 @@ void removeLowCardinalityInplace(Block & block) } } -void removeLowCardinalityInplace(Block & block, const Names & names) +void removeLowCardinalityInplace(Block & block, const Names & names, bool change_type) { for (const String & column_name : names) { auto & col = block.getByName(column_name); col.column = recursiveRemoveLowCardinality(col.column); - col.type = recursiveRemoveLowCardinality(col.type); + if (change_type) + col.type = recursiveRemoveLowCardinality(col.type); + } +} + +void restoreLowCardinalityInplace(Block & block) +{ + for (size_t i = 0; i < block.columns(); ++i) + { + auto & col = block.getByPosition(i); + if (col.type->lowCardinality() && col.column && !col.column->lowCardinality()) + col.column = changeLowCardinality(col.column, col.type->createColumn()); } } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 11fecd4e3fb..cfd727704a0 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -23,7 +23,8 @@ Columns materializeColumns(const Block & block, const Names & names); ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names); ColumnRawPtrs getRawPointers(const Columns & columns); void removeLowCardinalityInplace(Block & block); -void removeLowCardinalityInplace(Block & block, const Names & names); +void removeLowCardinalityInplace(Block & block, const Names & names, bool change_type = true); +void restoreLowCardinalityInplace(Block & block); ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_names_right); diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference new file mode 100644 index 00000000000..d1b29b46df6 --- /dev/null +++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference @@ -0,0 +1,29 @@ +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql new file mode 100644 index 00000000000..8a1601e3faa --- /dev/null +++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql @@ -0,0 +1,48 @@ +SET join_algorithm = 'partial_merge'; + +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS nr; + +CREATE TABLE t (`x` UInt32, `lc` LowCardinality(String)) ENGINE = Memory; +CREATE TABLE nr (`x` Nullable(UInt32), `lc` Nullable(String)) ENGINE = Memory; + +INSERT INTO t VALUES (1, 'l'); +INSERT INTO nr VALUES (2, NULL); + +SET join_use_nulls = 0; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SET join_use_nulls = 1; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; diff --git a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference new file mode 100644 index 00000000000..0612b4ca23e --- /dev/null +++ b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference @@ -0,0 +1,35 @@ +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +- +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +- +1 l \N LowCardinality(String) Nullable(String) +0 \N LowCardinality(String) Nullable(String) +0 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +- +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +0 \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +- +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +- +\N \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +\N \N Nullable(String) LowCardinality(String) +- +1 l \N LowCardinality(String) Nullable(String) +\N \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +\N \N LowCardinality(String) Nullable(String) +- diff --git a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql new file mode 100644 index 00000000000..2507613f051 --- /dev/null +++ b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql @@ -0,0 +1,65 @@ +SET join_algorithm = 'auto'; +SET max_bytes_in_join = 100; + +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS nr; + +CREATE TABLE t (`x` UInt32, `s` LowCardinality(String)) ENGINE = Memory; +CREATE TABLE nr (`x` Nullable(UInt32), `s` Nullable(String)) ENGINE = Memory; + +INSERT INTO t VALUES (1, 'l'); +INSERT INTO nr VALUES (2, NULL); + +SET join_use_nulls = 0; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (s) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (s) ORDER BY t.x; + +SET join_use_nulls = 1; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (s) ORDER BY t.x; + +SELECT '-'; + +-- TODO +-- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (s) ORDER BY t.x; +-- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (s) ORDER BY t.x; +-- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (s) ORDER BY t.x; + +DROP TABLE t; +DROP TABLE nr; From ba12e68d70fada79a2a3f503defe4fe5c8144fe8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 22 Sep 2020 13:23:10 +0300 Subject: [PATCH 261/390] Update insert_values_with_expressions.xml --- tests/performance/insert_values_with_expressions.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/insert_values_with_expressions.xml b/tests/performance/insert_values_with_expressions.xml index 3456cd0ec68..daa3488e34b 100644 --- a/tests/performance/insert_values_with_expressions.xml +++ b/tests/performance/insert_values_with_expressions.xml @@ -17,7 +17,7 @@ file('test_some_expr_matches.values', Values, 'i Int64, ari Array(Int64), ars Array(String)') - select * from file('test_all_expr_matches.values', Values, 'd DateTime, i UInt32, s String, ni Nullable(UInt64), ns Nullable(String), ars Array(String)') - select * from file('test_some_expr_matches.values', Values, 'i Int64, ari Array(Int64), ars Array(String)') + select * from file('test_all_expr_matches.values', Values, 'd DateTime, i UInt32, s String, ni Nullable(UInt64), ns Nullable(String), ars Array(String)') format Null + select * from file('test_some_expr_matches.values', Values, 'i Int64, ari Array(Int64), ars Array(String)') format Null From e646aa306f8356ab91fe6b3b2df91dab4a5863e6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 22 Sep 2020 13:25:16 +0300 Subject: [PATCH 262/390] Update columns_hashing.xml --- tests/performance/columns_hashing.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index c45e5d1fe39..3ea2e013acc 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -4,7 +4,7 @@ hits_100m_single - select sum(UserID + 1 in (select UserID from hits_100m_single)) from hits_100m_single + select sum(UserID + 1 in (select UserID from hits_10m_single)) from hits_10m_single select sum((UserID + 1, RegionID) in (select UserID, RegionID from hits_10m_single)) from hits_10m_single select sum(URL in (select URL from hits_10m_single where URL != '')) from hits_10m_single select sum(MobilePhoneModel in (select MobilePhoneModel from hits_100m_single where MobilePhoneModel != '')) from hits_100m_single From e97c9b16a7105841a31aaf31fa6b5398c7cf6443 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 22 Sep 2020 13:35:28 +0300 Subject: [PATCH 263/390] more diagnostics in fuzzer --- programs/client/Client.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 7c6d386ba05..d900eb17d78 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1167,6 +1167,9 @@ private: dump_of_cloned_ast.str().c_str()); fprintf(stderr, "dump after fuzz:\n"); fuzz_base->dumpTree(std::cerr); + + fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); + assert(false); } From def110f8355d24a83143885743af079d51343220 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 22 Sep 2020 19:05:23 +0800 Subject: [PATCH 264/390] Fix ASTWithElement clone. --- src/Parsers/ASTWithElement.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parsers/ASTWithElement.cpp b/src/Parsers/ASTWithElement.cpp index e8dd4ff0498..9d22286c2fd 100644 --- a/src/Parsers/ASTWithElement.cpp +++ b/src/Parsers/ASTWithElement.cpp @@ -6,6 +6,7 @@ namespace DB ASTPtr ASTWithElement::clone() const { const auto res = std::make_shared(*this); + res->children.clear(); res->name = name; res->subquery = subquery->clone(); res->children.emplace_back(res->subquery); From 2e6edc250c9d8899b31d890a9aea99d61883fa34 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 22 Sep 2020 14:35:46 +0300 Subject: [PATCH 265/390] better min/max for Int128 --- base/common/arithmeticOverflow.h | 8 ++++---- base/common/extended_types.h | 3 +++ base/common/itoa.h | 2 +- src/Core/AccurateComparison.h | 9 +++++---- src/DataTypes/DataTypesDecimal.h | 5 +++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/base/common/arithmeticOverflow.h b/base/common/arithmeticOverflow.h index c20fd635924..8df037a14af 100644 --- a/base/common/arithmeticOverflow.h +++ b/base/common/arithmeticOverflow.h @@ -31,8 +31,8 @@ namespace common template <> inline bool addOverflow(__int128 x, __int128 y, __int128 & res) { - static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; - static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + static constexpr __int128 min_int128 = minInt128(); + static constexpr __int128 max_int128 = maxInt128(); res = x + y; return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y); } @@ -79,8 +79,8 @@ namespace common template <> inline bool subOverflow(__int128 x, __int128 y, __int128 & res) { - static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; - static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + static constexpr __int128 min_int128 = minInt128(); + static constexpr __int128 max_int128 = maxInt128(); res = x - y; return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y); } diff --git a/base/common/extended_types.h b/base/common/extended_types.h index fe5f7184954..ea475163f6a 100644 --- a/base/common/extended_types.h +++ b/base/common/extended_types.h @@ -13,6 +13,9 @@ using wUInt256 = wide::integer<256, unsigned>; static_assert(sizeof(wInt256) == 32); static_assert(sizeof(wUInt256) == 32); +static constexpr __int128 minInt128() { return static_cast(1) << 127; } +static constexpr __int128 maxInt128() { return (static_cast(1) << 127) - 1; } + /// The standard library type traits, such as std::is_arithmetic, with one exception /// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior. /// So instead of using the std type_traits, we use our own version which allows extension. diff --git a/base/common/itoa.h b/base/common/itoa.h index 5d660ca4378..a02e7b68c05 100644 --- a/base/common/itoa.h +++ b/base/common/itoa.h @@ -372,7 +372,7 @@ static inline char * writeLeadingMinus(char * pos) static inline char * writeSIntText(int128_t x, char * pos) { - static const int128_t min_int128 = int128_t(0x8000000000000000ll) << 64; + static constexpr int128_t min_int128 = uint128_t(1) << 127; if (unlikely(x == min_int128)) { diff --git a/src/Core/AccurateComparison.h b/src/Core/AccurateComparison.h index bbd820bc65f..500346872db 100644 --- a/src/Core/AccurateComparison.h +++ b/src/Core/AccurateComparison.h @@ -4,6 +4,7 @@ #include #include "Defines.h" #include "Types.h" +#include #include #include @@ -382,8 +383,8 @@ inline bool equalsOp(DB::Float32 f, DB::UInt128 u) inline bool NO_SANITIZE_UNDEFINED greaterOp(DB::Int128 i, DB::Float64 f) { - static constexpr Int128 min_int128 = Int128(0x8000000000000000ll) << 64; - static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + static constexpr Int128 min_int128 = minInt128(); + static constexpr Int128 max_int128 = maxInt128(); if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) return static_cast(i) > f; @@ -394,8 +395,8 @@ inline bool NO_SANITIZE_UNDEFINED greaterOp(DB::Int128 i, DB::Float64 f) inline bool NO_SANITIZE_UNDEFINED greaterOp(DB::Float64 f, DB::Int128 i) { - static constexpr Int128 min_int128 = Int128(0x8000000000000000ll) << 64; - static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + static constexpr Int128 min_int128 = minInt128(); + static constexpr Int128 max_int128 = maxInt128(); if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) return f > static_cast(i); diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index fd7c1f91c68..079812c2e74 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -153,8 +153,9 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale) auto out = value * static_cast(DecimalUtils::scaleMultiplier(scale)); if constexpr (std::is_same_v) { - static constexpr Int128 min_int128 = Int128(0x8000000000000000ll) << 64; - static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + static constexpr Int128 min_int128 = minInt128(); + static constexpr Int128 max_int128 = maxInt128(); + if (out <= static_cast(min_int128) || out >= static_cast(max_int128)) throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range", ErrorCodes::DECIMAL_OVERFLOW); From 85cf3e798a075ea42c14b868dc1fa911b2ae06fe Mon Sep 17 00:00:00 2001 From: maqroll Date: Tue, 22 Sep 2020 15:05:43 +0200 Subject: [PATCH 266/390] decimal field in column definition packet shouldn't have size 1? --- src/Core/MySQL/PacketsProtocolText.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Core/MySQL/PacketsProtocolText.cpp b/src/Core/MySQL/PacketsProtocolText.cpp index 766bcf636e4..ad34cd8c28d 100644 --- a/src/Core/MySQL/PacketsProtocolText.cpp +++ b/src/Core/MySQL/PacketsProtocolText.cpp @@ -77,7 +77,7 @@ ColumnDefinition::ColumnDefinition( size_t ColumnDefinition::getPayloadSize() const { - return 13 + getLengthEncodedStringSize("def") + getLengthEncodedStringSize(schema) + getLengthEncodedStringSize(table) + getLengthEncodedStringSize(org_table) + \ + return 12 + getLengthEncodedStringSize("def") + getLengthEncodedStringSize(schema) + getLengthEncodedStringSize(table) + getLengthEncodedStringSize(org_table) + \ getLengthEncodedStringSize(name) + getLengthEncodedStringSize(org_name) + getLengthEncodedNumberSize(next_length); } @@ -96,7 +96,7 @@ void ColumnDefinition::readPayloadImpl(ReadBuffer & payload) payload.readStrict(reinterpret_cast(&column_length), 4); payload.readStrict(reinterpret_cast(&column_type), 1); payload.readStrict(reinterpret_cast(&flags), 2); - payload.readStrict(reinterpret_cast(&decimals), 2); + payload.readStrict(reinterpret_cast(&decimals), 1); payload.ignore(2); } @@ -113,7 +113,7 @@ void ColumnDefinition::writePayloadImpl(WriteBuffer & buffer) const buffer.write(reinterpret_cast(&column_length), 4); buffer.write(reinterpret_cast(&column_type), 1); buffer.write(reinterpret_cast(&flags), 2); - buffer.write(reinterpret_cast(&decimals), 2); + buffer.write(reinterpret_cast(&decimals), 1); writeChar(0x0, 2, buffer); } From 693f46655cd7e39c7a819f94aa5834b2c23a5381 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 22 Sep 2020 16:23:37 +0300 Subject: [PATCH 267/390] Update CHANGELOG.md --- CHANGELOG.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1764f07acf..d2a3ff036bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,74 @@ +## ClickHouse release 20.9 + +### ClickHouse release v20.9.2.20-stable, 2020-09-22 + +#### New Feature + +* Added util for database generation by query. [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973) ([ZeDRoman](https://github.com/ZeDRoman)). +* Added an aggregate function RankCorrelationSpearman which simply computes a rank correlation coefficient. [#11769](https://github.com/ClickHouse/ClickHouse/pull/11769) ([antikvist](https://github.com/antikvist)) [#14411](https://github.com/ClickHouse/ClickHouse/pull/14411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added column transformers, which can be applied to selected columns. For example, you can write `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). +* Added table function `view` which turns an subquery into a table object. This helps passing queries around. For instance, it can be used in remote/cluster table functions. [#12567](https://github.com/ClickHouse/ClickHouse/pull/12567) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix + +* Fixed incorrect comparison with primary key of type `FixedString`. This fixes https://github.com/ClickHouse/ClickHouse/issues/14908. [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* If function `bar` was called with specifically crafter arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes https://github.com/ClickHouse/ClickHouse/issues/14923. [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). +* Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare error in `SELECT` queries when the queried column has `DEFAULT` expression which depends on the other column which also has `DEFAULT` and not present in select query and not exists on disk. Partially fixes [#14531](https://github.com/ClickHouse/ClickHouse/issues/14531). [#14845](https://github.com/ClickHouse/ClickHouse/pull/14845) ([alesapin](https://github.com/alesapin)). +* Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([tavplubix](https://github.com/tavplubix)). +* Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Added the checker as neither calling `lc->isNullable()` nor calling `ls->getDictionaryPtr()->isNullable()` would return the correct result. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike](https://github.com/myrrc)). +* Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Proxy restart/start/stop/reload of SysVinit to systemd (if it is used). [#14460](https://github.com/ClickHouse/ClickHouse/pull/14460) ([Azat Khuzhin](https://github.com/azat)). +* Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). +* Fix exception during ALTER LIVE VIEW query with REFRESH command. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix QueryPlan lifetime (for EXPLAIN PIPELINE graph=1) for queries with nested interpreter. [#14315](https://github.com/ClickHouse/ClickHouse/pull/14315) ([Azat Khuzhin](https://github.com/azat)). +* Better check for tuple size in SSD cache complex key external dictionaries. This fixes [#13981](https://github.com/ClickHouse/ClickHouse/issues/13981). [#14313](https://github.com/ClickHouse/ClickHouse/pull/14313) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Disallows `CODEC` on `ALIAS` column type. Fixes [#13911](https://github.com/ClickHouse/ClickHouse/issues/13911). [#14263](https://github.com/ClickHouse/ClickHouse/pull/14263) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). + +#### Improvement + +* Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). +* Disallow empty time_zone argument in `toStartOf*` type of functions. [#14509](https://github.com/ClickHouse/ClickHouse/pull/14509) ([Bharat Nallan](https://github.com/bharatnc)). +* MySQL handler returns `OK` for queries like `SET @@var = value`. Such statement is ignored. It is needed because some MySQL drivers send `SET @@` query for setup after handshake https://github.com/ClickHouse/ClickHouse/issues/9336#issuecomment-686222422 . [#14469](https://github.com/ClickHouse/ClickHouse/pull/14469) ([BohuTANG](https://github.com/BohuTANG)). +* Now TTLs will be applied during merge if they were not previously materialized. [#14438](https://github.com/ClickHouse/ClickHouse/pull/14438) ([alesapin](https://github.com/alesapin)). +* Creating sets for multiple `JOIN` and `IN` in parallel. It may slightly improve performance for queries with several different `IN subquery` expressions. [#14412](https://github.com/ClickHouse/ClickHouse/pull/14412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now obfuscator supports UUID type as proposed in [#13163](https://github.com/ClickHouse/ClickHouse/issues/13163). [#14409](https://github.com/ClickHouse/ClickHouse/pull/14409) ([dimarub2000](https://github.com/dimarub2000)). +* Added new setting system_events_show_zero_values as proposed in [#11384](https://github.com/ClickHouse/ClickHouse/issues/11384). [#14404](https://github.com/ClickHouse/ClickHouse/pull/14404) ([dimarub2000](https://github.com/dimarub2000)). +* Implicitly convert primary key to not null in MaterializeMySQL (Same as MySQL). Fixes [#14114](https://github.com/ClickHouse/ClickHouse/issues/14114). [#14397](https://github.com/ClickHouse/ClickHouse/pull/14397) ([Winter Zhang](https://github.com/zhang2014)). +* Replace wide integers from boost multiprecision with implementation from https://github.com/cerevra/int. [#14229](https://github.com/ClickHouse/ClickHouse/pull/14229) ([Artem Zuikov](https://github.com/4ertus2)). +* Add default compression codec for parts in `system.part_log` with name `default_compression_codec`. [#14116](https://github.com/ClickHouse/ClickHouse/pull/14116) ([alesapin](https://github.com/alesapin)). +* Improve the Kafka engine performance by providing independent thread for each consumer. Separate thread pool for streaming engines (like Kafka). [#13939](https://github.com/ClickHouse/ClickHouse/pull/13939) ([fastio](https://github.com/fastio)). +* Add precision argument for DateTime type. [#13761](https://github.com/ClickHouse/ClickHouse/pull/13761) ([Winter Zhang](https://github.com/zhang2014)). +* Added requirepass authorization for Redis external dictionary. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Improvements in StorageRabbitMQ: Added connection and channels failure handling, proper commits, insert failures handling, better exchanges, queue durability and queue resume opportunity, new queue settings. Fixed tests. [#12761](https://github.com/ClickHouse/ClickHouse/pull/12761) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support custom codecs in compact parts. [#12183](https://github.com/ClickHouse/ClickHouse/pull/12183) ([Anton Popov](https://github.com/CurtizJ)). + +#### Performance Improvement + +* Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key (under optimize_skip_unused_shards and optimize_distributed_group_by_sharding_key). [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)). + +#### Build/Testing/Packaging Improvement + +* Lower binary size in debug build by removing debug info from `Functions`. This is needed only for one internal project in Yandex who is using very old linker. [#14549](https://github.com/ClickHouse/ClickHouse/pull/14549) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Prepare for build with clang 11. [#14455](https://github.com/ClickHouse/ClickHouse/pull/14455) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix the logic in backport script. In previous versions it was triggered for any labels of 100% red color. It was strange. [#14433](https://github.com/ClickHouse/ClickHouse/pull/14433) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Integration tests use default base config. All config changes are explicit with main_configs, user_configs and dictionaries parameters for instance. [#13647](https://github.com/ClickHouse/ClickHouse/pull/13647) ([Ilya Yatsishin](https://github.com/qoega)). + + + ## ClickHouse release 20.8 ### ClickHouse release v20.8.2.3-stable, 2020-09-08 From c9cfdffcd7df7b3c565cf955264b64e72d562a79 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 22 Sep 2020 16:52:56 +0300 Subject: [PATCH 268/390] mongo uri --- tests/integration/helpers/cluster.py | 4 ++-- .../test_mongo_uri.py | 11 ++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6d0f038daed..0b7fa9264bd 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -486,8 +486,8 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - connection.database_names() - print "Connected to Mongo dbs:", connection.database_names() + connection.list_database_names() + print "Connected to Mongo dbs:", connection.list_database_names() return except Exception as ex: print "Can't connect to Mongo " + str(ex) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index 53edfc4bbd2..5c09627d0b9 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -69,14 +69,7 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +# See comment in SourceMongoURI +@pytest.mark.parametrize("layout_name", ["flat"]) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) - -@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) -def test_complex(started_cluster, layout_name): - complex_tester.execute(layout_name, node) - -@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) -def test_ranged(started_cluster, layout_name): - ranged_tester.execute(layout_name, node) From 01237170afc4ce9d3a1873416ce241a4425fc4c9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 22 Sep 2020 17:25:48 +0300 Subject: [PATCH 269/390] Fix fast test CMakeOptions --- docker/test/fasttest/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ccbadb84f27..ca691e5303c 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -83,7 +83,7 @@ SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/submodule_log.txt -export CMAKE_LIBS_CONFIG="-DENABLE_LIBRARIES=0 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_THINLTO=0 -DUSE_UNWIND=1" +CMAKE_LIBS_CONFIG=(-DENABLE_LIBRARIES=0 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_THINLTO=0 -DUSE_UNWIND=1) export CCACHE_DIR=/ccache export CCACHE_BASEDIR=/ClickHouse @@ -96,7 +96,7 @@ ccache --zero-stats ||: mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "$CMAKE_LIBS_CONFIG" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/cmake_log.txt +cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/cmake_log.txt ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt ninja install | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/install_log.txt From 7fcf20e48cb861f5fcf585faad9ee23396a0e29d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 22 Sep 2020 14:56:40 +0300 Subject: [PATCH 270/390] enable more tests --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Storages/StorageDictionary.cpp | 11 ++- src/Storages/StorageDictionary.h | 4 +- .../test_atomic_drop_table/test.py | 2 +- tests/integration/test_backup_restore/test.py | 2 +- .../test.py | 2 +- .../test_cluster_copier/task0_description.xml | 4 +- .../task_month_to_week_description.xml | 4 +- .../task_test_block_size.xml | 4 +- tests/integration/test_cluster_copier/test.py | 21 +++-- .../test_cluster_copier/trivial_test.py | 2 +- .../test_dictionaries_dependency/test.py | 15 ++-- .../test_distributed_format/test.py | 2 +- .../test.py | 2 +- .../test_filesystem_layout/test.py | 16 ++++ tests/integration/test_partition/test.py | 2 +- .../test_polymorphic_parts/test.py | 2 +- tests/integration/test_system_merges/test.py | 2 +- .../01018_ddl_dictionaries_create.reference | 6 +- .../01018_ddl_dictionaries_create.sql | 78 +++++++++---------- .../01018_ddl_dictionaries_select.sql | 2 +- .../01018_ddl_dictionaries_special.sql | 2 +- .../01018_dictionaries_from_dictionaries.sql | 2 +- .../01033_dictionaries_lifetime.sql | 4 +- .../01037_polygon_dicts_correctness_all.sh | 2 +- .../01037_polygon_dicts_correctness_fast.sh | 2 +- .../01037_polygon_dicts_simple_functions.sh | 2 +- ...ionary_invalidate_query_switchover_long.sh | 2 +- .../01041_create_dictionary_if_not_exists.sql | 2 +- ...em_reload_dictionary_reloads_completely.sh | 2 +- ...dictionary_attribute_properties_values.sql | 2 +- .../0_stateless/01048_exists_query.sql | 2 +- .../01053_drop_database_mat_view.sql | 2 +- .../0_stateless/01053_ssd_dictionary.sql | 2 + .../01115_join_with_dictionary.sql | 2 +- .../0_stateless/01190_full_attach_syntax.sql | 2 +- .../01224_no_superfluous_dict_reload.sql | 2 +- ...1225_show_create_table_from_dictionary.sql | 2 +- .../01249_bad_arguments_for_bloom_filter.sql | 2 +- .../01251_dict_is_in_infinite_loop.sql | 2 +- .../01259_dictionary_custom_settings_ddl.sql | 4 +- .../01280_ssd_complex_key_dictionary.sql | 2 + ...20_create_sync_race_condition_zookeeper.sh | 4 +- .../0_stateless/01391_join_on_dict_crash.sql | 2 +- 44 files changed, 130 insertions(+), 107 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8d67672612c..db2ba9090ec 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -525,7 +525,7 @@ static bool allowDictJoin(StoragePtr joined_storage, const Context & context, St if (!dict) return false; - dict_name = dict->dictionaryName(); + dict_name = dict->resolvedDictionaryName(); auto dictionary = context.getExternalDictionariesLoader().getDictionary(dict_name); if (!dictionary) return false; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 99645d09d00..e859baa702e 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -92,6 +92,12 @@ String StorageDictionary::generateNamesAndTypesDescription(const NamesAndTypesLi return ss.str(); } +String StorageDictionary::resolvedDictionaryName() const +{ + if (location == Location::SameDatabaseAndNameAsDictionary) + return dictionary_name; + return DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); +} StorageDictionary::StorageDictionary( const StorageID & table_id_, @@ -100,9 +106,6 @@ StorageDictionary::StorageDictionary( Location location_) : IStorage(table_id_) , dictionary_name(dictionary_name_) - , resolved_dictionary_name(location_ == Location::SameDatabaseAndNameAsDictionary - ? dictionary_name - : DatabaseCatalog::instance().resolveDictionaryName(dictionary_name)) , location(location_) { StorageInMemoryMetadata storage_metadata; @@ -135,7 +138,7 @@ Pipe StorageDictionary::read( const size_t max_block_size, const unsigned /*threads*/) { - auto dictionary = context.getExternalDictionariesLoader().getDictionary(resolved_dictionary_name); + auto dictionary = context.getExternalDictionariesLoader().getDictionary(resolvedDictionaryName()); auto stream = dictionary->getBlockInputStream(column_names, max_block_size); /// TODO: update dictionary interface for processors. return Pipe(std::make_shared(stream)); diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 528ee6533b7..5c7beb88d88 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -28,7 +28,8 @@ public: static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); static String generateNamesAndTypesDescription(const NamesAndTypesList & list); - const String & dictionaryName() const { return resolved_dictionary_name; } + const String & dictionaryName() const { return dictionary_name; } + String resolvedDictionaryName() const; /// Specifies where the table is located relative to the dictionary. enum class Location @@ -50,7 +51,6 @@ public: private: const String dictionary_name; - const String resolved_dictionary_name; const Location location; protected: diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index 7ff06c7f369..dc1ad47aa75 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -13,7 +13,7 @@ node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_ def start_cluster(): try: cluster.start() - node1.query("CREATE DATABASE zktest ENGINE=Ordinary;") + node1.query("CREATE DATABASE zktest ENGINE=Ordinary;") # Different behaviour with Atomic node1.query( ''' CREATE TABLE zktest.atomic_drop_table (n UInt32) diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index 111dc6d24f8..170266aaaea 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -14,7 +14,7 @@ path_to_data = '/var/lib/clickhouse/' def started_cluster(): try: cluster.start() - q('CREATE DATABASE test ENGINE = Ordinary') + q('CREATE DATABASE test ENGINE = Ordinary') # Different path in shadow/ with Atomic yield cluster diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 45404c1170c..5ed1cb06787 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -142,7 +142,7 @@ def test_backup_from_old_version_config(started_cluster): def test_backup_and_alter(started_cluster): - node4.query("CREATE DATABASE test ENGINE=Ordinary") + node4.query("CREATE DATABASE test ENGINE=Ordinary") # Different path in shadow/ with Atomic node4.query("CREATE TABLE test.backup_table(A Int64, B String, C Date) Engine = MergeTree order by tuple()") diff --git a/tests/integration/test_cluster_copier/task0_description.xml b/tests/integration/test_cluster_copier/task0_description.xml index d56053ffd39..72eff8d464d 100644 --- a/tests/integration/test_cluster_copier/task0_description.xml +++ b/tests/integration/test_cluster_copier/task0_description.xml @@ -33,7 +33,7 @@ 3 4 5 6 1 2 0 - ENGINE=ReplicatedMergeTree PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 + ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 d + 1 @@ -93,4 +93,4 @@ - + \ No newline at end of file diff --git a/tests/integration/test_cluster_copier/task_month_to_week_description.xml b/tests/integration/test_cluster_copier/task_month_to_week_description.xml index 26dfc7d3e00..ee134603310 100644 --- a/tests/integration/test_cluster_copier/task_month_to_week_description.xml +++ b/tests/integration/test_cluster_copier/task_month_to_week_description.xml @@ -34,7 +34,7 @@ ENGINE= - ReplicatedMergeTree + ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/b', '{replica}') PARTITION BY toMonday(date) ORDER BY d @@ -97,4 +97,4 @@ - + \ No newline at end of file diff --git a/tests/integration/test_cluster_copier/task_test_block_size.xml b/tests/integration/test_cluster_copier/task_test_block_size.xml index c9c99a083ea..ea63d580c1c 100644 --- a/tests/integration/test_cluster_copier/task_test_block_size.xml +++ b/tests/integration/test_cluster_copier/task_test_block_size.xml @@ -28,7 +28,7 @@ ENGINE= - ReplicatedMergeTree + ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/test_block_size', '{replica}') ORDER BY d PARTITION BY partition @@ -99,4 +99,4 @@ - + \ No newline at end of file diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 88dac06f158..2a9e696ca46 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -81,11 +81,11 @@ class Task1: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( cluster_num)) ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + - "ENGINE=ReplicatedMergeTree " + + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") @@ -110,11 +110,10 @@ class Task1: class Task2: - def __init__(self, cluster, unique_zk_path): + def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read() - self.unique_zk_path = unique_zk_path def start(self): instance = cluster.instances['s0_0_0'] @@ -122,13 +121,11 @@ class Task2: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( cluster_num)) ddl_check_query(instance, - "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) " - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/" + self.unique_zk_path + "', " - "'{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") + "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") @@ -172,7 +169,7 @@ class Task_test_block_size: ddl_check_query(instance, """ CREATE TABLE test_block_size ON CLUSTER shard_0_0 (partition Date, d UInt64) - ENGINE=ReplicatedMergeTree + ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) instance.query( @@ -335,17 +332,17 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs @pytest.mark.timeout(600) def test_copy_month_to_week_partition(started_cluster): - execute_task(Task2(started_cluster, "test1"), []) + execute_task(Task2(started_cluster), []) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering(started_cluster): - execute_task(Task2(started_cluster, "test2"), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): - execute_task(Task2(started_cluster, "test3"), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) def test_block_size(started_cluster): diff --git a/tests/integration/test_cluster_copier/trivial_test.py b/tests/integration/test_cluster_copier/trivial_test.py index 035faf0bb9f..3d0c5d0f5b0 100644 --- a/tests/integration/test_cluster_copier/trivial_test.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -59,7 +59,7 @@ class TaskTrivial: for node in [source, destination]: node.query("DROP DATABASE IF EXISTS default") - node.query("CREATE DATABASE IF NOT EXISTS default") + node.query("CREATE DATABASE IF NOT EXISTS default ENGINE=Ordinary") source.query("CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1) " "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index 9c36da229e1..d615f90dc79 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -12,15 +12,18 @@ def start_cluster(): try: cluster.start() for node in nodes: - node.query("CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS test") + # Different internal dictionary name with Atomic + node.query("CREATE DATABASE IF NOT EXISTS test_ordinary ENGINE=Ordinary") node.query("CREATE DATABASE IF NOT EXISTS atest") node.query("CREATE DATABASE IF NOT EXISTS ztest") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") node.query("INSERT INTO test.source VALUES (5,6)") - node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \ - "LAYOUT(FLAT()) LIFETIME(0)") + for db in ("test", "test_ordinary"): + node.query("CREATE DICTIONARY {}.dict(x UInt64, y UInt64) PRIMARY KEY x " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \ + "LAYOUT(FLAT()) LIFETIME(0)".format(db)) yield cluster finally: @@ -91,10 +94,10 @@ def test_dependency_via_explicit_table(node): def test_dependency_via_dictionary_database(node): node.query("CREATE DATABASE dict_db ENGINE=Dictionary") - d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] + d_names = ["test_ordinary.adict", "test_ordinary.zdict", "atest.dict", "ztest.dict"] for d_name in d_names: node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) " \ + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_ordinary.dict' DB 'dict_db')) " \ "LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) def check(): diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index 7658814a720..7e9d740c171 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -15,7 +15,7 @@ cluster_param = pytest.mark.parametrize("cluster", [ def started_cluster(): try: cluster.start() - node.query("create database test engine=Ordinary") + node.query("create database test") yield cluster finally: diff --git a/tests/integration/test_distributed_storage_configuration/test.py b/tests/integration/test_distributed_storage_configuration/test.py index a932e9a55c5..d293b96399d 100644 --- a/tests/integration/test_distributed_storage_configuration/test.py +++ b/tests/integration/test_distributed_storage_configuration/test.py @@ -17,7 +17,7 @@ node = cluster.add_instance('node', def start_cluster(): try: cluster.start() - node.query('CREATE DATABASE test ENGINE=Ordinary') + node.query('CREATE DATABASE test ENGINE=Ordinary') # Different paths with Atomic yield cluster finally: cluster.shutdown() diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index e2441d0d20d..2519d0e5ac3 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -27,3 +27,19 @@ def test_file_path_escaping(started_cluster): node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) node.exec_in_container( ["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) + +def test_file_path_escaping_atomic_db(started_cluster): + node.query('CREATE DATABASE IF NOT EXISTS `test 2` ENGINE = Atomic') + node.query(''' + CREATE TABLE `test 2`.`T.a_b,l-e!` UUID '12345678-1000-4000-8000-000000000001' (`~Id` UInt32) + ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id` SETTINGS min_bytes_for_wide_part = 0; + ''') + node.query('''INSERT INTO `test 2`.`T.a_b,l-e!` VALUES (1);''') + node.query('''ALTER TABLE `test 2`.`T.a_b,l-e!` FREEZE;''') + + node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin"]) + # Check symlink + node.exec_in_container(["bash", "-c", "test -L /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21"]) + node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) + node.exec_in_container( + ["bash", "-c", "test -f /var/lib/clickhouse/shadow/2/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin"]) diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 5b27ff94ddb..679c6fb8c5b 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -13,7 +13,7 @@ path_to_data = '/var/lib/clickhouse/' def started_cluster(): try: cluster.start() - q('CREATE DATABASE test ENGINE = Ordinary') + q('CREATE DATABASE test ENGINE = Ordinary') # Different path in shadow/ with Atomic yield cluster diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 39745e4f3f0..50a8192fbc5 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -515,7 +515,7 @@ def test_in_memory_alters(start_cluster): def test_polymorphic_parts_index(start_cluster): - node1.query('CREATE DATABASE test_index ENGINE=Ordinary') + node1.query('CREATE DATABASE test_index ENGINE=Ordinary') # Different paths with Atomic node1.query(''' CREATE TABLE test_index.index_compact(a UInt32, s String) ENGINE = MergeTree ORDER BY a diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 07e6f7331d9..1f2da606cd1 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -21,7 +21,7 @@ node2 = cluster.add_instance('node2', def started_cluster(): try: cluster.start() - node1.query('CREATE DATABASE test ENGINE=Ordinary') + node1.query('CREATE DATABASE test ENGINE=Ordinary') # Different paths with Atomic node2.query('CREATE DATABASE test ENGINE=Ordinary') yield cluster diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference index 7c2eca9cedf..5b020911d2e 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference @@ -1,14 +1,14 @@ =DICTIONARY in Ordinary DB -CREATE DICTIONARY ordinary_db.dict1\n(\n `key_column` UInt64 DEFAULT 0,\n `second_column` UInt8 DEFAULT 1,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) +CREATE DICTIONARY db_01018.dict1\n(\n `key_column` UInt64 DEFAULT 0,\n `second_column` UInt8 DEFAULT 1,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict_01018\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) dict1 1 -ordinary_db dict1 +db_01018 dict1 ==DETACH DICTIONARY 0 ==ATTACH DICTIONARY dict1 1 -ordinary_db dict1 +db_01018 dict1 ==DROP DICTIONARY 0 =DICTIONARY in Memory DB diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql index d7d7c02baa8..3261b1e61d1 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql @@ -1,12 +1,12 @@ SET send_logs_level = 'fatal'; -DROP DATABASE IF EXISTS database_for_dict; +DROP DATABASE IF EXISTS database_for_dict_01018; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict_01018; -DROP TABLE IF EXISTS database_for_dict.table_for_dict; +DROP TABLE IF EXISTS database_for_dict_01018.table_for_dict; -CREATE TABLE database_for_dict.table_for_dict +CREATE TABLE database_for_dict_01018.table_for_dict ( key_column UInt64, second_column UInt8, @@ -15,64 +15,64 @@ CREATE TABLE database_for_dict.table_for_dict ENGINE = MergeTree() ORDER BY key_column; -INSERT INTO database_for_dict.table_for_dict VALUES (1, 100, 'Hello world'); +INSERT INTO database_for_dict_01018.table_for_dict VALUES (1, 100, 'Hello world'); -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01018; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE db_01018; SELECT '=DICTIONARY in Ordinary DB'; -DROP DICTIONARY IF EXISTS ordinary_db.dict1; +DROP DICTIONARY IF EXISTS db_01018.dict1; -CREATE DICTIONARY ordinary_db.dict1 +CREATE DICTIONARY db_01018.dict1 ( key_column UInt64 DEFAULT 0, second_column UInt8 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()); -SHOW CREATE DICTIONARY ordinary_db.dict1; +SHOW CREATE DICTIONARY db_01018.dict1; -SHOW DICTIONARIES FROM ordinary_db LIKE 'dict1'; +SHOW DICTIONARIES FROM db_01018 LIKE 'dict1'; -EXISTS DICTIONARY ordinary_db.dict1; +EXISTS DICTIONARY db_01018.dict1; SELECT database, name FROM system.dictionaries WHERE name LIKE 'dict1'; SELECT '==DETACH DICTIONARY'; -DETACH DICTIONARY ordinary_db.dict1; +DETACH DICTIONARY db_01018.dict1; -SHOW DICTIONARIES FROM ordinary_db LIKE 'dict1'; +SHOW DICTIONARIES FROM db_01018 LIKE 'dict1'; -EXISTS DICTIONARY ordinary_db.dict1; +EXISTS DICTIONARY db_01018.dict1; SELECT database, name FROM system.dictionaries WHERE name LIKE 'dict1'; SELECT '==ATTACH DICTIONARY'; -ATTACH DICTIONARY ordinary_db.dict1; +ATTACH DICTIONARY db_01018.dict1; -SHOW DICTIONARIES FROM ordinary_db LIKE 'dict1'; +SHOW DICTIONARIES FROM db_01018 LIKE 'dict1'; -EXISTS DICTIONARY ordinary_db.dict1; +EXISTS DICTIONARY db_01018.dict1; SELECT database, name FROM system.dictionaries WHERE name LIKE 'dict1'; SELECT '==DROP DICTIONARY'; -DROP DICTIONARY IF EXISTS ordinary_db.dict1; +DROP DICTIONARY IF EXISTS db_01018.dict1; -SHOW DICTIONARIES FROM ordinary_db LIKE 'dict1'; +SHOW DICTIONARIES FROM db_01018 LIKE 'dict1'; -EXISTS DICTIONARY ordinary_db.dict1; +EXISTS DICTIONARY db_01018.dict1; SELECT database, name FROM system.dictionaries WHERE name LIKE 'dict1'; -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01018; DROP DATABASE IF EXISTS memory_db; @@ -87,7 +87,7 @@ CREATE DICTIONARY memory_db.dict2 third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()); -- {serverError 48} @@ -112,7 +112,7 @@ CREATE DICTIONARY lazy_db.dict3 third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column, second_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(COMPLEX_KEY_HASHED()); -- {serverError 48} @@ -120,45 +120,45 @@ DROP DATABASE IF EXISTS lazy_db; SELECT '=DROP DATABASE WITH DICTIONARY'; -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01018; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE db_01018; -CREATE DICTIONARY ordinary_db.dict4 +CREATE DICTIONARY db_01018.dict4 ( key_column UInt64 DEFAULT 0, second_column UInt8 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()); -SHOW DICTIONARIES FROM ordinary_db; +SHOW DICTIONARIES FROM db_01018; -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01018; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE db_01018; -SHOW DICTIONARIES FROM ordinary_db; +SHOW DICTIONARIES FROM db_01018; -CREATE DICTIONARY ordinary_db.dict4 +CREATE DICTIONARY db_01018.dict4 ( key_column UInt64 DEFAULT 0, second_column UInt8 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()); -SHOW DICTIONARIES FROM ordinary_db; +SHOW DICTIONARIES FROM db_01018; -DROP DATABASE IF EXISTS ordinary_db; +DROP DATABASE IF EXISTS db_01018; -DROP TABLE IF EXISTS database_for_dict.table_for_dict; +DROP TABLE IF EXISTS database_for_dict_01018.table_for_dict; -DROP DATABASE IF EXISTS database_for_dict; +DROP DATABASE IF EXISTS database_for_dict_01018; DROP DATABASE IF EXISTS memory_db; diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql index f4de269e774..4b548a913ea 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql @@ -2,7 +2,7 @@ SET send_logs_level = 'fatal'; DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; CREATE TABLE database_for_dict.table_for_dict ( diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_special.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_special.sql index 6d9b499a247..ede5897bdf7 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_special.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_special.sql @@ -2,7 +2,7 @@ SET send_logs_level = 'fatal'; DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; SELECT '***date dict***'; diff --git a/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.sql b/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.sql index 4d2cd6351b5..86180643f88 100644 --- a/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.sql +++ b/tests/queries/0_stateless/01018_dictionaries_from_dictionaries.sql @@ -2,7 +2,7 @@ SET send_logs_level = 'fatal'; DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; CREATE TABLE database_for_dict.table_for_dict ( diff --git a/tests/queries/0_stateless/01033_dictionaries_lifetime.sql b/tests/queries/0_stateless/01033_dictionaries_lifetime.sql index 57776e1fec1..0a8288c2df0 100644 --- a/tests/queries/0_stateless/01033_dictionaries_lifetime.sql +++ b/tests/queries/0_stateless/01033_dictionaries_lifetime.sql @@ -2,7 +2,7 @@ SET send_logs_level = 'fatal'; DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; DROP TABLE IF EXISTS database_for_dict.table_for_dict; @@ -19,7 +19,7 @@ INSERT INTO database_for_dict.table_for_dict VALUES (1, 100, 'Hello world'); DROP DATABASE IF EXISTS ordinary_db; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE ordinary_db; DROP DICTIONARY IF EXISTS ordinary_db.dict1; diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh index 1b80fcef80b..e7df8282433 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_all.sh @@ -11,7 +11,7 @@ tar -xf "${CURDIR}"/01037_test_data_search.tar.gz -C "${CURDIR}" $CLICKHOUSE_CLIENT -n --query=" DROP DATABASE IF EXISTS test_01037; -CREATE DATABASE test_01037 Engine = Ordinary; +CREATE DATABASE test_01037; DROP TABLE IF EXISTS test_01037.points; CREATE TABLE test_01037.points (x Float64, y Float64) ENGINE = Memory; " diff --git a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh index 4ca95b72937..22d08d425a6 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_correctness_fast.sh @@ -11,7 +11,7 @@ tar -xf "${CURDIR}"/01037_test_data_perf.tar.gz -C "${CURDIR}" $CLICKHOUSE_CLIENT -n --query=" DROP DATABASE IF EXISTS test_01037; -CREATE DATABASE test_01037 Engine = Ordinary; +CREATE DATABASE test_01037; DROP TABLE IF EXISTS test_01037.points; CREATE TABLE test_01037.points (x Float64, y Float64) ENGINE = Memory; " diff --git a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh index d32b75ca735..c3d820e1292 100755 --- a/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh +++ b/tests/queries/0_stateless/01037_polygon_dicts_simple_functions.sh @@ -8,7 +8,7 @@ TMP_DIR="/tmp" $CLICKHOUSE_CLIENT -n --query=" DROP DATABASE IF EXISTS test_01037; -CREATE DATABASE test_01037 Engine = Ordinary; +CREATE DATABASE test_01037; DROP TABLE IF EXISTS test_01037.polygons_array; diff --git a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh index 44a192cf178..6b509ac7925 100755 --- a/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh +++ b/tests/queries/0_stateless/01040_dictionary_invalidate_query_switchover_long.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS dictdb" -$CLICKHOUSE_CLIENT --query "CREATE DATABASE dictdb Engine = Ordinary" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE dictdb" $CLICKHOUSE_CLIENT --query " CREATE TABLE dictdb.dict_invalidate diff --git a/tests/queries/0_stateless/01041_create_dictionary_if_not_exists.sql b/tests/queries/0_stateless/01041_create_dictionary_if_not_exists.sql index 8c30abeb28f..5ec76e6ae91 100644 --- a/tests/queries/0_stateless/01041_create_dictionary_if_not_exists.sql +++ b/tests/queries/0_stateless/01041_create_dictionary_if_not_exists.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS dictdb.table_for_dict; DROP DICTIONARY IF EXISTS dictdb.dict_exists; DROP DATABASE IF EXISTS dictdb; -CREATE DATABASE dictdb ENGINE = Ordinary; +CREATE DATABASE dictdb; CREATE TABLE dictdb.table_for_dict ( diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh index f03f7511a4f..46031a3d508 100755 --- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh +++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh @@ -8,7 +8,7 @@ set -e -o pipefail # Run the client. $CLICKHOUSE_CLIENT --multiquery <<'EOF' DROP DATABASE IF EXISTS dictdb; -CREATE DATABASE dictdb Engine = Ordinary; +CREATE DATABASE dictdb; CREATE TABLE dictdb.table(x Int64, y Int64, insert_time DateTime) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO dictdb.table VALUES (12, 102, now()); diff --git a/tests/queries/0_stateless/01043_dictionary_attribute_properties_values.sql b/tests/queries/0_stateless/01043_dictionary_attribute_properties_values.sql index afd1c1c5780..adeb5630529 100644 --- a/tests/queries/0_stateless/01043_dictionary_attribute_properties_values.sql +++ b/tests/queries/0_stateless/01043_dictionary_attribute_properties_values.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS dictdb; -CREATE DATABASE dictdb Engine = Ordinary; +CREATE DATABASE dictdb; CREATE TABLE dictdb.dicttbl(key Int64, value_default String, value_expression String) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO dictdb.dicttbl VALUES (12, 'hello', '55:66:77'); diff --git a/tests/queries/0_stateless/01048_exists_query.sql b/tests/queries/0_stateless/01048_exists_query.sql index 700b4f5983d..31b6d2af6c0 100644 --- a/tests/queries/0_stateless/01048_exists_query.sql +++ b/tests/queries/0_stateless/01048_exists_query.sql @@ -3,7 +3,7 @@ EXISTS TABLE db_01048.t_01048; EXISTS DICTIONARY db_01048.t_01048; DROP DATABASE IF EXISTS db_01048; -CREATE DATABASE db_01048 Engine = Ordinary; +CREATE DATABASE db_01048; DROP TABLE IF EXISTS db_01048.t_01048; EXISTS db_01048.t_01048; diff --git a/tests/queries/0_stateless/01053_drop_database_mat_view.sql b/tests/queries/0_stateless/01053_drop_database_mat_view.sql index 60803bced7e..9f7438d594e 100644 --- a/tests/queries/0_stateless/01053_drop_database_mat_view.sql +++ b/tests/queries/0_stateless/01053_drop_database_mat_view.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS some_tests; -CREATE DATABASE some_tests ENGINE=Ordinary; +CREATE DATABASE some_tests ENGINE=Ordinary; -- Different inner table name with Atomic create table some_tests.my_table ENGINE = MergeTree(day, (day), 8192) as select today() as day, 'mystring' as str; show tables from some_tests; diff --git a/tests/queries/0_stateless/01053_ssd_dictionary.sql b/tests/queries/0_stateless/01053_ssd_dictionary.sql index 416d26bd637..fb4acdeadb4 100644 --- a/tests/queries/0_stateless/01053_ssd_dictionary.sql +++ b/tests/queries/0_stateless/01053_ssd_dictionary.sql @@ -23,6 +23,8 @@ INSERT INTO database_for_dict.table_for_dict SELECT number, 0, -1, 'c' FROM syst DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; +-- FIXME filesystem error: in create_directory: Permission denied [/var/lib/clickhouse] +-- Probably we need rewrite it to integration test CREATE DICTIONARY database_for_dict.ssd_dict ( id UInt64, diff --git a/tests/queries/0_stateless/01115_join_with_dictionary.sql b/tests/queries/0_stateless/01115_join_with_dictionary.sql index f1477df7df2..807b53c39c0 100644 --- a/tests/queries/0_stateless/01115_join_with_dictionary.sql +++ b/tests/queries/0_stateless/01115_join_with_dictionary.sql @@ -1,7 +1,7 @@ SET send_logs_level = 'fatal'; DROP DATABASE IF EXISTS db_01115; -CREATE DATABASE db_01115 Engine = Ordinary; +CREATE DATABASE db_01115; USE db_01115; diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql index 225e66bed5c..78f0f53d101 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.sql +++ b/tests/queries/0_stateless/01190_full_attach_syntax.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS test_01190; -CREATE DATABASE test_01190 ENGINE=Ordinary; +CREATE DATABASE test_01190 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01190; CREATE TABLE test_01190.table_for_dict (key UInt64, col UInt8) ENGINE = Memory; diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql index 55689411ad6..da4928a26fb 100644 --- a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS dict_db_01224; DROP DATABASE IF EXISTS dict_db_01224_dictionary; -CREATE DATABASE dict_db_01224 ENGINE=Ordinary; +CREATE DATABASE dict_db_01224 ENGINE=Ordinary; -- Different internal dictionary name with Atomic CREATE DATABASE dict_db_01224_dictionary Engine=Dictionary; CREATE TABLE dict_db_01224.dict_data (key UInt64, val UInt64) Engine=Memory(); diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql index 8980a9fd70b..24d10537dbb 100644 --- a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS dict_db_01225; DROP DATABASE IF EXISTS dict_db_01225_dictionary; -CREATE DATABASE dict_db_01225 ENGINE=Ordinary; +CREATE DATABASE dict_db_01225 ENGINE=Ordinary; -- Different internal dictionary name with Atomic CREATE DATABASE dict_db_01225_dictionary Engine=Dictionary; CREATE TABLE dict_db_01225.dict_data (key UInt64, val UInt64) Engine=Memory(); diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql index 7bf334582a2..8902b164c09 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS test_01249; -CREATE DATABASE test_01249 ENGINE=Ordinary; +CREATE DATABASE test_01249 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01249; CREATE TABLE bloom_filter_idx_good(`u64` UInt64, `i32` Int32, `f64` Float64, `d` Decimal(10, 2), `s` String, `e` Enum8('a' = 1, 'b' = 2, 'c' = 3), `dt` Date, INDEX bloom_filter_a i32 TYPE bloom_filter(0, 1) GRANULARITY 1) ENGINE = MergeTree() ORDER BY u64 SETTINGS index_granularity = 8192; -- { serverError 42 } diff --git a/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql index decf65dc8cf..8e7e76697b5 100644 --- a/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql +++ b/tests/queries/0_stateless/01251_dict_is_in_infinite_loop.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; DROP TABLE IF EXISTS database_for_dict.dict_source; CREATE TABLE database_for_dict.dict_source (id UInt64, parent_id UInt64, value String) ENGINE = Memory; diff --git a/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql index cbac234305d..9c2174c8469 100644 --- a/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql +++ b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql @@ -1,6 +1,6 @@ DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +CREATE DATABASE database_for_dict; DROP TABLE IF EXISTS database_for_dict.table_for_dict; @@ -17,7 +17,7 @@ INSERT INTO database_for_dict.table_for_dict VALUES (100500, 10000000, 'Hello wo DROP DATABASE IF EXISTS ordinary_db; -CREATE DATABASE ordinary_db ENGINE = Ordinary; +CREATE DATABASE ordinary_db; DROP DICTIONARY IF EXISTS ordinary_db.dict1; diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql index 952a8c2ff55..9faafb6c0c7 100644 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql @@ -24,6 +24,8 @@ INSERT INTO database_for_dict.table_for_dict SELECT toString(number), number + 1 DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; +-- FIXME filesystem error: in create_directory: Permission denied [/var/lib/clickhouse] +-- Probably we need rewrite it to integration test CREATE DICTIONARY database_for_dict.ssd_dict ( k1 String, diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index ce1f08b4c51..f72c9da27ba 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01320" -$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Different bahaviour of DROP with Atomic function thread1() { @@ -26,4 +26,4 @@ timeout 10 bash -c thread2 & wait -$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01320" +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01320" 2>&1 | grep -v "New table appeared in database being dropped or detached. Try again." || exit 0 diff --git a/tests/queries/0_stateless/01391_join_on_dict_crash.sql b/tests/queries/0_stateless/01391_join_on_dict_crash.sql index 998e0e21745..238a966727f 100644 --- a/tests/queries/0_stateless/01391_join_on_dict_crash.sql +++ b/tests/queries/0_stateless/01391_join_on_dict_crash.sql @@ -1,5 +1,5 @@ DROP DATABASE IF EXISTS db_01391; -CREATE DATABASE db_01391 Engine = Ordinary; +CREATE DATABASE db_01391; USE db_01391; DROP TABLE IF EXISTS t; From 0636e249a106a1d3287470938583fe490d296b33 Mon Sep 17 00:00:00 2001 From: myrrc Date: Tue, 22 Sep 2020 17:53:14 +0300 Subject: [PATCH 271/390] fix: path traversal --- docs/tools/cmake_in_clickhouse_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 9ac6e9475d8..614ac87882c 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -65,7 +65,7 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No entities[name] = path, formatted_entity def process_file(root_path: str, input_name: str) -> None: - with open(root_path + input_name, 'r') as cmake_file: + with open(os.path.join(root_path, input_name), 'r') as cmake_file: contents: str = cmake_file.read() def get_line_and_comment(target: str) -> Tuple[int, str]: @@ -94,7 +94,7 @@ def process_folder(root_path:str, name: str) -> None: for root, _, files in os.walk(name): for f in files: if f == "CMakeLists.txt" or ".cmake" in f: - process_file(root_path, root + "/" + f) + process_file(root_path, os.path.join(root, f)) def generate_cmake_flags_files(root_path: str) -> None: output_file_name: str = root_path + "docs/en/development/cmake-in-clickhouse.md" From 476b48830118a68a67bbd1e1de3aa0be33d5b2b9 Mon Sep 17 00:00:00 2001 From: myrrc Date: Tue, 22 Sep 2020 18:08:48 +0300 Subject: [PATCH 272/390] fix: resulting files path --- docs/tools/cmake_in_clickhouse_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 614ac87882c..066ffe08df1 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -97,9 +97,9 @@ def process_folder(root_path:str, name: str) -> None: process_file(root_path, os.path.join(root, f)) def generate_cmake_flags_files(root_path: str) -> None: - output_file_name: str = root_path + "docs/en/development/cmake-in-clickhouse.md" - header_file_name: str = root_path + "docs/_includes/cmake_in_clickhouse_header.md" - footer_file_name: str = root_path + "docs/_includes/cmake_in_clickhouse_footer.md" + output_file_name: str = os.path.join(root_path, "docs/en/development/cmake-in-clickhouse.md") + header_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_header.md") + footer_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_footer.md") process_file(root_path, "CMakeLists.txt") process_file(root_path, "programs/CMakeLists.txt") From 0752fa01371d07c340217239b1fe6450f1e5684f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 22 Sep 2020 18:52:25 +0300 Subject: [PATCH 273/390] Rethrow copy of exception in SSD dictionaries A follow-up to https://github.com/ClickHouse/ClickHouse/pull/14689 , that should fix asan reports in stress test in master. --- src/Dictionaries/CacheDictionary.cpp | 3 ++- src/Dictionaries/SSDCacheDictionary.cpp | 20 +++++++++++++++++-- .../SSDComplexKeyCacheDictionary.cpp | 20 +++++++++++++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index cb39dffeb6c..27fe99a93ec 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -835,7 +835,8 @@ void CacheDictionary::waitForCurrentUpdateFinish(UpdateUnitPtr & update_unit_ptr catch (...) { throw DB::Exception(ErrorCodes::CACHE_DICTIONARY_UPDATE_FAIL, - "Dictionary update failed: {}", + "Update failed for dictionary '{}': {}", + getDictionaryID().getNameForLogs(), getCurrentExceptionMessage(true /*with stack trace*/, true /*check embedded stack trace*/)); } diff --git a/src/Dictionaries/SSDCacheDictionary.cpp b/src/Dictionaries/SSDCacheDictionary.cpp index 20e62acbd82..5547e34758f 100644 --- a/src/Dictionaries/SSDCacheDictionary.cpp +++ b/src/Dictionaries/SSDCacheDictionary.cpp @@ -53,6 +53,7 @@ namespace ErrorCodes extern const int AIO_READ_ERROR; extern const int AIO_WRITE_ERROR; extern const int BAD_ARGUMENTS; + extern const int CACHE_DICTIONARY_UPDATE_FAIL; extern const int CANNOT_ALLOCATE_MEMORY; extern const int CANNOT_CREATE_DIRECTORY; extern const int CANNOT_FSYNC; @@ -1193,8 +1194,23 @@ void SSDCacheStorage::update(DictionarySourcePtr & source_ptr, const std::vector { /// TODO: use old values - /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`. - std::rethrow_exception(last_update_exception); + // We don't have expired data for that `id` so all we can do is + // to rethrow `last_exception`. We might have to throw the same + // exception for different callers of dictGet() in different + // threads, which might then modify the exception object, so we + // have to throw a copy. + try + { + std::rethrow_exception(last_update_exception); + } + catch (...) + { + throw DB::Exception(ErrorCodes::CACHE_DICTIONARY_UPDATE_FAIL, + "Update failed for dictionary '{}': {}", + getPath(), + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } } /// Set key diff --git a/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp b/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp index 972d10da24d..44847df48ff 100644 --- a/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp +++ b/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp @@ -54,6 +54,7 @@ namespace ErrorCodes extern const int AIO_READ_ERROR; extern const int AIO_WRITE_ERROR; extern const int BAD_ARGUMENTS; + extern const int CACHE_DICTIONARY_UPDATE_FAIL; extern const int CANNOT_ALLOCATE_MEMORY; extern const int CANNOT_CREATE_DIRECTORY; extern const int CANNOT_FSYNC; @@ -1266,8 +1267,23 @@ void SSDComplexKeyCacheStorage::update( { /// TODO: use old values. - /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`. - std::rethrow_exception(last_update_exception); + // We don't have expired data for that `id` so all we can do is + // to rethrow `last_exception`. We might have to throw the same + // exception for different callers of dictGet() in different + // threads, which might then modify the exception object, so we + // have to throw a copy. + try + { + std::rethrow_exception(last_update_exception); + } + catch (...) + { + throw DB::Exception(ErrorCodes::CACHE_DICTIONARY_UPDATE_FAIL, + "Update failed for dictionary '{}': {}", + getPath(), + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } } std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; From f80ffc4dce7166ac7381bc2927021055a10b8dad Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 22 Sep 2020 18:56:24 +0300 Subject: [PATCH 274/390] restore decimal_casts perf test --- tests/performance/{broken => }/decimal_casts.xml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/performance/{broken => }/decimal_casts.xml (100%) diff --git a/tests/performance/broken/decimal_casts.xml b/tests/performance/decimal_casts.xml similarity index 100% rename from tests/performance/broken/decimal_casts.xml rename to tests/performance/decimal_casts.xml From 4b795c7b6a9468c6b1b61f2fd8c487af84f8c3e3 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 22 Sep 2020 18:58:14 +0300 Subject: [PATCH 275/390] temporarily run test with wide parts only --- .../configs/config.d/storage_configuration.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml index 131219abf3d..f1b92ab32a6 100644 --- a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml @@ -24,5 +24,9 @@ + + + 0 + From 74f3c592c78c41216187f6812bd33ea7c15c7844 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 22 Sep 2020 19:20:37 +0300 Subject: [PATCH 276/390] update comment after #14940 --- src/Core/Defines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 3a7d29e92b1..ba3d37242fa 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -70,7 +70,7 @@ /// Mininum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 -/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. +/// Version of ClickHouse TCP protocol. Increment it manually when you change the protocol. #define DBMS_TCP_PROTOCOL_VERSION 54441 /// The boundary on which the blocks for asynchronous file operations should be aligned. From b1db8ed2b0dee339ebaa3e0bbd259dc77d5e95c9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 20:42:11 +0300 Subject: [PATCH 277/390] Update CHANGELOG.md --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2a3ff036bf..68569d2b949 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,9 @@ #### New Feature -* Added util for database generation by query. [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973) ([ZeDRoman](https://github.com/ZeDRoman)). -* Added an aggregate function RankCorrelationSpearman which simply computes a rank correlation coefficient. [#11769](https://github.com/ClickHouse/ClickHouse/pull/11769) ([antikvist](https://github.com/antikvist)) [#14411](https://github.com/ClickHouse/ClickHouse/pull/14411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Added column transformers, which can be applied to selected columns. For example, you can write `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). -* Added table function `view` which turns an subquery into a table object. This helps passing queries around. For instance, it can be used in remote/cluster table functions. [#12567](https://github.com/ClickHouse/ClickHouse/pull/12567) ([Amos Bird](https://github.com/amosbird)). +* Added column transformers `EXCEPT`, `REPLACE`, `APPLY`, which can be applied to the list of selected columns (after `*` or `COLUMNS(...)`). For example, you can write `SELECT * EXCEPT(URL) REPLACE(number + 1 AS number)`. Another example: `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). +* Added an aggregate function `rankCorr` which computes a rank correlation coefficient. [#11769](https://github.com/ClickHouse/ClickHouse/pull/11769) ([antikvist](https://github.com/antikvist)) [#14411](https://github.com/ClickHouse/ClickHouse/pull/14411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added table function `view` which turns a subquery into a table object. This helps passing queries around. For instance, it can be used in remote/cluster table functions. [#12567](https://github.com/ClickHouse/ClickHouse/pull/12567) ([Amos Bird](https://github.com/amosbird)). #### Bug Fix @@ -38,6 +37,10 @@ * Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). * Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). +#### Experimental Feature + +* Added a tool for random database generation by given SELECT query. [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973) ([ZeDRoman](https://github.com/ZeDRoman)). + #### Improvement * Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). From 4398efb2eed1dcb4ca70bcd777da90d534e88553 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 20:49:32 +0300 Subject: [PATCH 278/390] Update CHANGELOG.md --- CHANGELOG.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68569d2b949..cba8fae8640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,12 @@ #### Bug Fix -* Fixed incorrect comparison with primary key of type `FixedString`. This fixes https://github.com/ClickHouse/ClickHouse/issues/14908. [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). -* If function `bar` was called with specifically crafter arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed inconsistent comparison with primary key of type `FixedString` on index analysis if they're compered with a string of less size. This fixes https://github.com/ClickHouse/ClickHouse/issues/14908. [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes https://github.com/ClickHouse/ClickHouse/issues/14923. [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). * Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). * Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([alexey-milovidov](https://github.com/alexey-milovidov)). @@ -19,27 +23,23 @@ * Fix rare error in `SELECT` queries when the queried column has `DEFAULT` expression which depends on the other column which also has `DEFAULT` and not present in select query and not exists on disk. Partially fixes [#14531](https://github.com/ClickHouse/ClickHouse/issues/14531). [#14845](https://github.com/ClickHouse/ClickHouse/pull/14845) ([alesapin](https://github.com/alesapin)). * Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). * Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([tavplubix](https://github.com/tavplubix)). -* Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). -* Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). -* Added the checker as neither calling `lc->isNullable()` nor calling `ls->getDictionaryPtr()->isNullable()` would return the correct result. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike](https://github.com/myrrc)). -* Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). -* Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). -* Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix possibly incorrect result of function `has` when LowCardinality and Nullable types are involved. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike](https://github.com/myrrc)). +* Cleanup data directory after Zookeeper exceptions during CREATE query for tables with ReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix rare segfaults in functions with combinator `-Resample`, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafted parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Proxy restart/start/stop/reload of SysVinit to systemd (if it is used). [#14460](https://github.com/ClickHouse/ClickHouse/pull/14460) ([Azat Khuzhin](https://github.com/azat)). -* Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). * Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). -* Fix exception during ALTER LIVE VIEW query with REFRESH command. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix exception during ALTER LIVE VIEW query with REFRESH command. LIVE VIEW is an experimental feature. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). * Fix QueryPlan lifetime (for EXPLAIN PIPELINE graph=1) for queries with nested interpreter. [#14315](https://github.com/ClickHouse/ClickHouse/pull/14315) ([Azat Khuzhin](https://github.com/azat)). * Better check for tuple size in SSD cache complex key external dictionaries. This fixes [#13981](https://github.com/ClickHouse/ClickHouse/issues/13981). [#14313](https://github.com/ClickHouse/ClickHouse/pull/14313) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Disallows `CODEC` on `ALIAS` column type. Fixes [#13911](https://github.com/ClickHouse/ClickHouse/issues/13911). [#14263](https://github.com/ClickHouse/ClickHouse/pull/14263) ([Bharat Nallan](https://github.com/bharatnc)). * Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). +* Fix arrayJoin() capturing in lambda (exception with logical error message was thrown). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). #### Experimental Feature -* Added a tool for random database generation by given SELECT query. [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973) ([ZeDRoman](https://github.com/ZeDRoman)). +* Added `db-generator` tool for random database generation by given SELECT queries. It may faciliate reproducing issues when there is only incomplete bug report from the user. [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973) ([ZeDRoman](https://github.com/ZeDRoman)). #### Improvement From 71feb111d20c505d1528b35f5d0e98627e94e081 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 20:52:43 +0300 Subject: [PATCH 279/390] Update CHANGELOG.md --- CHANGELOG.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cba8fae8640..8e4acdc293f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,21 +47,21 @@ * Disallow empty time_zone argument in `toStartOf*` type of functions. [#14509](https://github.com/ClickHouse/ClickHouse/pull/14509) ([Bharat Nallan](https://github.com/bharatnc)). * MySQL handler returns `OK` for queries like `SET @@var = value`. Such statement is ignored. It is needed because some MySQL drivers send `SET @@` query for setup after handshake https://github.com/ClickHouse/ClickHouse/issues/9336#issuecomment-686222422 . [#14469](https://github.com/ClickHouse/ClickHouse/pull/14469) ([BohuTANG](https://github.com/BohuTANG)). * Now TTLs will be applied during merge if they were not previously materialized. [#14438](https://github.com/ClickHouse/ClickHouse/pull/14438) ([alesapin](https://github.com/alesapin)). -* Creating sets for multiple `JOIN` and `IN` in parallel. It may slightly improve performance for queries with several different `IN subquery` expressions. [#14412](https://github.com/ClickHouse/ClickHouse/pull/14412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Now obfuscator supports UUID type as proposed in [#13163](https://github.com/ClickHouse/ClickHouse/issues/13163). [#14409](https://github.com/ClickHouse/ClickHouse/pull/14409) ([dimarub2000](https://github.com/dimarub2000)). -* Added new setting system_events_show_zero_values as proposed in [#11384](https://github.com/ClickHouse/ClickHouse/issues/11384). [#14404](https://github.com/ClickHouse/ClickHouse/pull/14404) ([dimarub2000](https://github.com/dimarub2000)). -* Implicitly convert primary key to not null in MaterializeMySQL (Same as MySQL). Fixes [#14114](https://github.com/ClickHouse/ClickHouse/issues/14114). [#14397](https://github.com/ClickHouse/ClickHouse/pull/14397) ([Winter Zhang](https://github.com/zhang2014)). -* Replace wide integers from boost multiprecision with implementation from https://github.com/cerevra/int. [#14229](https://github.com/ClickHouse/ClickHouse/pull/14229) ([Artem Zuikov](https://github.com/4ertus2)). -* Add default compression codec for parts in `system.part_log` with name `default_compression_codec`. [#14116](https://github.com/ClickHouse/ClickHouse/pull/14116) ([alesapin](https://github.com/alesapin)). -* Improve the Kafka engine performance by providing independent thread for each consumer. Separate thread pool for streaming engines (like Kafka). [#13939](https://github.com/ClickHouse/ClickHouse/pull/13939) ([fastio](https://github.com/fastio)). -* Add precision argument for DateTime type. [#13761](https://github.com/ClickHouse/ClickHouse/pull/13761) ([Winter Zhang](https://github.com/zhang2014)). -* Added requirepass authorization for Redis external dictionary. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). -* Improvements in StorageRabbitMQ: Added connection and channels failure handling, proper commits, insert failures handling, better exchanges, queue durability and queue resume opportunity, new queue settings. Fixed tests. [#12761](https://github.com/ClickHouse/ClickHouse/pull/12761) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Now `clickhouse-obfuscator` supports UUID type as proposed in [#13163](https://github.com/ClickHouse/ClickHouse/issues/13163). [#14409](https://github.com/ClickHouse/ClickHouse/pull/14409) ([dimarub2000](https://github.com/dimarub2000)). +* Added new setting `system_events_show_zero_values` as proposed in [#11384](https://github.com/ClickHouse/ClickHouse/issues/11384). [#14404](https://github.com/ClickHouse/ClickHouse/pull/14404) ([dimarub2000](https://github.com/dimarub2000)). +* Implicitly convert primary key to not null in `MaterializeMySQL` (Same as `MySQL`). Fixes [#14114](https://github.com/ClickHouse/ClickHouse/issues/14114). [#14397](https://github.com/ClickHouse/ClickHouse/pull/14397) ([Winter Zhang](https://github.com/zhang2014)). +* Replace wide integers (256 bit) from boost multiprecision with implementation from https://github.com/cerevra/int. 256bit integers are experimental. [#14229](https://github.com/ClickHouse/ClickHouse/pull/14229) ([Artem Zuikov](https://github.com/4ertus2)). +* Add default compression codec for parts in `system.part_log` with the name `default_compression_codec`. [#14116](https://github.com/ClickHouse/ClickHouse/pull/14116) ([alesapin](https://github.com/alesapin)). +* Add precision argument for `DateTime` type. It allows to use `DateTime` name instead of `DateTime64`. [#13761](https://github.com/ClickHouse/ClickHouse/pull/13761) ([Winter Zhang](https://github.com/zhang2014)). +* Added requirepass authorization for `Redis` external dictionary. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Improvements in `RabbitMQ` engine: added connection and channels failure handling, proper commits, insert failures handling, better exchanges, queue durability and queue resume opportunity, new queue settings. Fixed tests. [#12761](https://github.com/ClickHouse/ClickHouse/pull/12761) ([Kseniia Sumarokova](https://github.com/kssenii)). * Support custom codecs in compact parts. [#12183](https://github.com/ClickHouse/ClickHouse/pull/12183) ([Anton Popov](https://github.com/CurtizJ)). #### Performance Improvement * Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key (under optimize_skip_unused_shards and optimize_distributed_group_by_sharding_key). [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)). +* Creating sets for multiple `JOIN` and `IN` in parallel. It may slightly improve performance for queries with several different `IN subquery` expressions. [#14412](https://github.com/ClickHouse/ClickHouse/pull/14412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Improve Kafka engine performance by providing independent thread for each consumer. Separate thread pool for streaming engines (like Kafka). [#13939](https://github.com/ClickHouse/ClickHouse/pull/13939) ([fastio](https://github.com/fastio)). #### Build/Testing/Packaging Improvement From cf24789c1b6fe1dc81443cbf1ce6b4b0f10dd97d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Sep 2020 21:25:31 +0300 Subject: [PATCH 280/390] Update 01505_log_distributed_deadlock.sql --- .../0_stateless/01505_log_distributed_deadlock.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql index b3f294df327..2b0b2b97188 100644 --- a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql +++ b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql @@ -1,7 +1,12 @@ -create table t_local(a int) engine Log; +DROP TABLE IF EXISTS t_local; +DROP TABLE IF EXISTS t_dist; -create table t_dist (a int) engine Distributed(test_shard_localhost, 'default', 't_local', cityHash64(a)); +create table t_local(a int) engine Log; +create table t_dist (a int) engine Distributed(test_shard_localhost, currentDatabase(), 't_local', cityHash64(a)); set insert_distributed_sync = 1; insert into t_dist values (1); + +DROP TABLE t_local; +DROP TABLE t_dist; From cc24ef9f83671c385801d676b095b30ddeb94abe Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Tue, 22 Sep 2020 21:35:29 +0200 Subject: [PATCH 281/390] Better debug message from MergeTreeDataSelectExecutor See #15168 --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ffd5d616cb0..2b8b886daaf 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -679,7 +679,7 @@ Pipe MergeTreeDataSelectExecutor::readFromParts( parts_with_ranges.resize(next_part); } - LOG_DEBUG(log, "Selected {} parts by date, {} parts by key, {} marks by primary key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks_pk.load(std::memory_order_relaxed), sum_marks, sum_ranges); + LOG_DEBUG(log, "Selected {} parts by partition key, {} parts by primary key, {} marks by primary key, {} marks to read from {} ranges", parts.size(), parts_with_ranges.size(), sum_marks_pk.load(std::memory_order_relaxed), sum_marks, sum_ranges); if (parts_with_ranges.empty()) return {}; From e2ee97202bf7b81cae3ab05458b6c4d8609c2617 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 22 Sep 2020 23:16:34 +0300 Subject: [PATCH 282/390] USE_LIBGSASL not required to use internal librdkafka --- cmake/find/rdkafka.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index d9f815dbcdd..ac11322f408 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -14,10 +14,10 @@ if (NOT ENABLE_RDKAFKA) return() endif() -if (NOT ARCH_ARM AND USE_LIBGSASL) +if (NOT ARCH_ARM) option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) elseif(USE_INTERNAL_RDKAFKA_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM} AND USE_LIBGSASL=${USE_LIBGSASL}") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM}") endif () if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt") From 88dc3126ad9f47b3f2406085994d4fa1547d4256 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 23 Sep 2020 00:26:59 +0300 Subject: [PATCH 283/390] Adjustments --- benchmark/hardware.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 0ff71df19ae..693a69deb1f 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -36,6 +36,14 @@ if [[ ! -f clickhouse ]]; then $FASTER_DOWNLOAD "$AMD64_BIN_URL" elif [[ $CPU == aarch64 ]]; then $FASTER_DOWNLOAD "$AARCH64_BIN_URL" + + # Download configs. ARM version has no embedded configs. + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml + mkdir config.d + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/access_control.xml -O config.d/access_control.xml + wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml else echo "Unsupported CPU type: $CPU" exit 1 @@ -85,7 +93,7 @@ cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do echo -n "[" for i in $(seq 1 $TRIES); do - RES=$(./clickhouse client --max_memory_usage 100000000000 --time --format=Null --query="$query" 2>&1) + RES=$(./clickhouse client --max_memory_usage 100000000000 --time --format=Null --query="$query" 2>&1 ||:) [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" [[ "$i" != $TRIES ]] && echo -n ", " done From f726bfa3a52010002ed0d9d1a84f174c94bf4d61 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 23 Sep 2020 03:26:15 +0300 Subject: [PATCH 284/390] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 161 ++++++++++++++++++++------ 1 file changed, 126 insertions(+), 35 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index fa371df2d4a..b1d56ef005e 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -97,7 +97,9 @@ Upd. Есть pull request. Upd. Сделано. Частный случай такой задачи уже есть в https://clickhouse.tech/docs/ru/operations/table_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить. -### 1.10. Пережатие старых данных в фоне {#perezhatie-starykh-dannykh-v-fone} +### 1.10. + Пережатие старых данных в фоне {#perezhatie-starykh-dannykh-v-fone} + +В master, сделал Александр Сапин, https://github.com/ClickHouse/ClickHouse/pull/14494 Будет делать Кирилл Барухов, ВШЭ, экспериментальная реализация к весне 2020. Нужно для Яндекс.Метрики. @@ -138,27 +140,32 @@ Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) ### 1.14. Не писать столбцы, полностью состоящие из нулей {#ne-pisat-stolbtsy-polnostiu-sostoiashchie-iz-nulei} -Антон Попов. Q3. +Антон Попов. Q4. В очереди. Простая задача, является небольшим пререквизитом для потенциальной поддержки полуструктурированных данных. +Upd. В очереди после чтения срезов столбцов. ### 1.15. Возможность иметь разный первичный ключ в разных кусках {#vozmozhnost-imet-raznyi-pervichnyi-kliuch-v-raznykh-kuskakh} Сложная задача, только после 1.3. Upd. В обсуждении. +Upd. Взял в работу Amos Bird. Описана концепция. Совпадает с 1.16. ### 1.16. Несколько физических представлений для одного куска данных {#neskolko-fizicheskikh-predstavlenii-dlia-odnogo-kuska-dannykh} Сложная задача, только после 1.3 и 1.6. Позволяет компенсировать 21.20. Upd. В обсуждении. +Upd. Взял в работу Amos Bird. Описана концепция, работа на начальной стадии. ### 1.17. Несколько сортировок для одной таблицы {#neskolko-sortirovok-dlia-odnoi-tablitsy} Сложная задача, только после 1.3 и 1.6. Upd. В обсуждении. +Upd. Взял в работу Amos Bird. Описана концепция. Совпадает с 1.16. -### 1.18. Отдельное хранение файлов кусков {#otdelnoe-khranenie-failov-kuskov} +### 1.18. - Отдельное хранение файлов кусков {#otdelnoe-khranenie-failov-kuskov} Требует 1.3 и 1.6. Полная замена hard links на sym links, что будет лучше для 1.12. +Отменено. ## 2. Крупные рефакторинги {#krupnye-refaktoringi} @@ -194,13 +201,14 @@ Upd. Старый код по большей части удалён. ### 2.5. Версионирование состояний агрегатных функций {#versionirovanie-sostoianii-agregatnykh-funktsii} -В очереди. +В очереди. Описана схема реализации. Алексей Миловидов. ### 2.6. Правая часть IN как тип данных. Выполнение IN в виде скалярного подзапроса {#pravaia-chast-in-kak-tip-dannykh-vypolnenie-in-v-vide-skaliarnogo-podzaprosa} Требует 2.1. +Отменено. -### 2.7. Нормализация Context {#normalizatsiia-context} +### 2.7. + Нормализация Context {#normalizatsiia-context} В очереди. Нужно для YQL. @@ -209,12 +217,14 @@ Upd. Старый код по большей части удалён. Upd. Каталог БД вынесен из Context. Upd. SharedContext вынесен из Context. Upd. Проблема нейтрализована и перестала быть актуальной. +Upd. Вообще всё стало Ок. ### 2.8. Декларативный парсер запросов {#deklarativnyi-parser-zaprosov} Средний приоритет. Нужно для YQL. Upd. В очереди. Иван Лежанкин. +Upd. Задача в финальной стадии. Пока рассматривается только как альтернативный парсер, описание которого подойдёт для сторонних приложений. ### 2.9. + Логгировние в format-стиле {#loggirovnie-v-format-stile} @@ -225,10 +235,12 @@ Upd. В очереди. Иван Лежанкин. ### 2.10. Запрашивать у таблиц не столбцы, а срезы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy} В очереди. +В работе, Антон Попов, Q4. -### 2.11. Разбирательство и нормализация функциональности для bitmap {#razbiratelstvo-i-normalizatsiia-funktsionalnosti-dlia-bitmap} +### 2.11. - Разбирательство и нормализация функциональности для bitmap {#razbiratelstvo-i-normalizatsiia-funktsionalnosti-dlia-bitmap} В очереди. +Не актуально. ### 2.12. Декларативные сигнатуры функций {#deklarativnye-signatury-funktsii} @@ -265,7 +277,7 @@ Upd. Поползновения наблюдаются. Требует 3.1. -### + 3.3. Исправить катастрофически отвратительно неприемлемый поиск по документации {#ispravit-katastroficheski-otvratitelno-nepriemlemyi-poisk-po-dokumentatsii} +### 3.3. + Исправить катастрофически отвратительно неприемлемый поиск по документации {#ispravit-katastroficheski-otvratitelno-nepriemlemyi-poisk-po-dokumentatsii} [Иван Блинков](https://github.com/blinkov/) - очень хороший человек. Сам сайт документации основан на технологиях, не удовлетворяющих требованиям задачи, и эти технологии трудно исправить. Задачу будет делать первый встретившийся нам frontend разработчик, которого мы сможем заставить это сделать. @@ -311,7 +323,6 @@ Upd. Сейчас обсуждается, как сделать другую з ### 4.8. Разделить background pool для fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge} В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. -Александр Казаков. ## 5. Операции {#operatsii} @@ -450,6 +461,7 @@ UBSan включен в функциональных тестах, но не в ### 7.12. Показывать тестовое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr} Пока есть просто показ тестового покрытия всего кода. +Отложено. ### 7.13. + Включение аналога -Weverything в gcc {#vkliuchenie-analoga-weverything-v-gcc} @@ -598,7 +610,7 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII Upd. Илья Яцишин сделал табличную функцию `generateRandom`. Upd. Эльдар Заитов добавляет OSS Fuzz. Upd. Сделаны randomString, randomFixedString. -Upd. Сделаны fuzzBits, fuzzBytes. +Upd. Сделаны fuzzBits. ### 7.24. Fuzzing лексера и парсера запросов; кодеков и форматов {#fuzzing-leksera-i-parsera-zaprosov-kodekov-i-formatov} @@ -649,10 +661,11 @@ Upd. В Аркадии частично работает небольшая ча В очереди. Нужно для Яндекс.Метрики. -### 7.32. Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov} +### 7.32. + Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov} Роман Ильговский. Нужно для Яндекс.Метрики. -Есть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973 +Есть pull request: https://github.com/ClickHouse/ClickHouse/pull/10973 +Готово. Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных. @@ -660,6 +673,8 @@ Upd. В Аркадии частично работает небольшая ча Обфускация запросов: имея секретные запросы и структуру таблиц, заменить имена полей и константы, чтобы запросы можно было использовать в качестве публично доступных тестов. +Upd. Последняя часть пока не сделана и будет сделана отдельно. + ### 7.33. Выкладывать патч релизы в репозиторий автоматически {#vykladyvat-patch-relizy-v-repozitorii-avtomaticheski} В очереди. Иван Лежанкин. @@ -701,10 +716,11 @@ Upd. Частично решён вопрос с visibility - есть како Altinity. Никто не делает эту задачу. -### 8.2. Поддержка Mongo Atlas URI {#podderzhka-mongo-atlas-uri} +### 8.2. - Поддержка Mongo Atlas URI {#podderzhka-mongo-atlas-uri} [Александр Кузьменков](https://github.com/akuzm). Upd. Задача взята в работу. +Все pull requests успешно закрыты. ### 8.3. + Доработки globs (правильная поддержка диапазонов, уменьшение числа одновременных stream-ов) {#dorabotki-globs-pravilnaia-podderzhka-diapazonov-umenshenie-chisla-odnovremennykh-stream-ov} @@ -721,6 +737,7 @@ Upd. Задача взята в работу. ### 8.6. Kerberos аутентификация для HDFS и Kafka {#kerberos-autentifikatsiia-dlia-hdfs-i-kafka} Андрей Коняев, ArenaData. Он куда-то пропал. +Upd. В процессе работа для Kafka. ### 8.7. + Исправление мелочи HDFS на очень старых ядрах Linux {#ispravlenie-melochi-hdfs-na-ochen-starykh-iadrakh-linux} @@ -759,6 +776,8 @@ Upd. В стадии код-ревью. ### 8.15. Запись данных в CapNProto {#zapis-dannykh-v-capnproto} +Отложено. + ### 8.16. + Поддержка формата Avro {#podderzhka-formata-avro} Andrew Onyshchuk. Есть pull request. Q1. Сделано. @@ -814,12 +833,13 @@ Upd. Готово. Низкий приоритет. Отменено. -### 8.21. Поддержка произвольного количества языков для имён регионов {#podderzhka-proizvolnogo-kolichestva-iazykov-dlia-imion-regionov} +### 8.21. - Поддержка произвольного количества языков для имён регионов {#podderzhka-proizvolnogo-kolichestva-iazykov-dlia-imion-regionov} Нужно для БК. Декабрь 2019. В декабре для БК сделан минимальный вариант этой задачи. Максимальный вариант, вроде, никому не нужен. Upd. Всё ещё кажется, что задача не нужна. +Отменено. ### 8.22. + Поддержка синтаксиса для переменных в стиле MySQL {#podderzhka-sintaksisa-dlia-peremennykh-v-stile-mysql} @@ -831,6 +851,7 @@ Upd. Сделано теми людьми, кому не запрещено ра ### 8.23. Подписка для импорта обновляемых и ротируемых логов в ФС {#podpiska-dlia-importa-obnovliaemykh-i-rotiruemykh-logov-v-fs} Желательно 2.15. +Отложено. ## 9. Безопасность {#bezopasnost} @@ -870,9 +891,10 @@ Upd. Одну причину устранили, но ещё что-то неи Upd. Нас заставляют переписать эту библиотеку с одного API на другое, так как старое внезапно устарело. Кажется, что переписывание случайно исправит все проблемы. Upd. Ура, нашли причину и исправили. -### 10.3. Возможность чтения данных из статических таблиц в YT словарях {#vozmozhnost-chteniia-dannykh-iz-staticheskikh-tablits-v-yt-slovariakh} +### 10.3. - Возможность чтения данных из статических таблиц в YT словарях {#vozmozhnost-chteniia-dannykh-iz-staticheskikh-tablits-v-yt-slovariakh} Нужно для БК и Метрики. +Отменено. ### 10.4. - Словарь из YDB (KikiMR) {#slovar-iz-ydb-kikimr} @@ -884,9 +906,11 @@ Upd. Ура, нашли причину и исправили. Для MySQL сделал Clément Rodriguez. -### 10.6. Словари из Cassandra и Couchbase {#slovari-iz-cassandra-i-couchbase} +### 10.6. + Словари из Cassandra и Couchbase {#slovari-iz-cassandra-i-couchbase} Готова Cassandra. +Couchbase отменён, так как не было спроса. +Aerospike под вопросом. ### 10.7. Поддержка Nullable в словарях {#podderzhka-nullable-v-slovariakh} @@ -929,10 +953,14 @@ Upd. Задача в финальной стадии готовности. ### 10.17. Локальный дамп состояния словаря для быстрого старта сервера {#lokalnyi-damp-sostoianiia-slovaria-dlia-bystrogo-starta-servera} +Отложено. + ### 10.18. Таблица Join или словарь на удалённом сервере как key-value БД для cache словаря {#tablitsa-join-ili-slovar-na-udalionnom-servere-kak-key-value-bd-dlia-cache-slovaria} ### 10.19. Возможность зарегистрировать некоторые функции, использующие словари, под пользовательскими именами {#vozmozhnost-zaregistrirovat-nekotorye-funktsii-ispolzuiushchie-slovari-pod-polzovatelskimi-imenami} +Отложено. + ## 11. Интерфейсы {#interfeisy} @@ -943,6 +971,7 @@ Upd. Задача в финальной стадии готовности. Нужно разобраться, как упаковывать Java в статический бинарник, возможно AppImage. Или предоставить максимально простую инструкцию по установке jdbc-bridge. Может быть будет заинтересован Александр Крашенинников, Badoo, так как он разработал jdbc-bridge. Upd. Александр Крашенинников перешёл в другую компанию и больше не занимается этим. +Upd. Задачу взял Zhichun Wu. ### 11.3. + Интеграционные тесты ODBC драйвера путём подключения ClickHouse к самому себе через ODBC {#integratsionnye-testy-odbc-draivera-putiom-podkliucheniia-clickhouse-k-samomu-sebe-cherez-odbc} @@ -960,6 +989,8 @@ Altinity целиком взяли на себя поддержку clickhouse-c ### 11.7. Интерактивный режим работы программы clickhouse-local {#interaktivnyi-rezhim-raboty-programmy-clickhouse-local} +Отложено. + ### 11.8. + Поддержка протокола PostgreSQL {#podderzhka-protokola-postgresql} Элбакян Мовсес Андраникович, ВШЭ. @@ -998,14 +1029,17 @@ Q1. Сделано управление правами полностью, но Аутентификация через LDAP - Денис Глазачев. [Виталий Баранов](https://github.com/vitlibar) и Денис Глазачев, Altinity. Требует 12.1. Q3. +Upd. Pull request на финальной стадии. ### 12.4. Подключение IDM системы Яндекса как справочника пользователей и прав доступа {#podkliuchenie-idm-sistemy-iandeksa-kak-spravochnika-polzovatelei-i-prav-dostupa} Пока низкий приоритет. Нужно для Метрики. Требует 12.3. +Отложено. ### 12.5. Pluggable аутентификация с помощью Kerberos (возможно, подключение GSASL) {#pluggable-autentifikatsiia-s-pomoshchiu-kerberos-vozmozhno-podkliuchenie-gsasl} [Виталий Баранов](https://github.com/vitlibar) и Денис Глазачев, Altinity. Требует 12.1. +Upd. Есть pull request. ### 12.6. + Информация о пользователях и квотах в системной таблице {#informatsiia-o-polzovateliakh-i-kvotakh-v-sistemnoi-tablitse} @@ -1033,6 +1067,7 @@ Q3. Upd. Не уследили, и задачу стали обсуждать менеджеры. Upd. Задачу смотрит Александр Казаков. Upd. Задача взята в работу. +Upd. Задача как будто взята в работу. ## 14. Диалект SQL {#dialekt-sql} @@ -1041,7 +1076,9 @@ Upd. Задача взята в работу. Нужно для DataLens. А также для внедрения в BI инструмент Looker. -### 14.2. Поддержка WITH для подзапросов {#podderzhka-with-dlia-podzaprosov} +### 14.2. + Поддержка WITH для подзапросов {#podderzhka-with-dlia-podzaprosov} + +Сделал Amos Bird. ### 14.3. Поддержка подстановок для множеств в правой части IN {#podderzhka-podstanovok-dlia-mnozhestv-v-pravoi-chasti-in} @@ -1057,11 +1094,13 @@ zhang2014 ### 14.6. Глобальный scope для WITH {#globalnyi-scope-dlia-with} +В обсуждении. Amos Bird. + ### 14.7. Nullable для WITH ROLLUP, WITH CUBE, WITH TOTALS {#nullable-dlia-with-rollup-with-cube-with-totals} Простая задача. -### 14.8. Модификаторы DISTINCT, ORDER BY для агрегатных функций {#modifikatory-distinct-order-by-dlia-agregatnykh-funktsii} +### 14.8. + Модификаторы DISTINCT, ORDER BY для агрегатных функций {#modifikatory-distinct-order-by-dlia-agregatnykh-funktsii} В ClickHouse поддерживается вычисление COUNT(DISTINCT x). Предлагается добавить возможность использования модификатора DISTINCT для всех агрегатных функций. Например, AVG(DISTINCT x) - вычислить среднее значение для всех различных значений x. Под вопросом вариант, в котором фильтрация уникальных значений выполняется по одному выражению, а агрегация по другому. @@ -1069,6 +1108,7 @@ zhang2014 Upd. Есть pull request-ы. Upd. DISTINCT готов. +Upd. ORDER BY отменён и будет заново сделан уже с LIMIT. ### 14.9. + Поддержка запроса EXPLAIN {#podderzhka-zaprosa-explain} @@ -1079,8 +1119,12 @@ Upd. Есть pull request. Готово. ### 14.11. Функции для grouping sets {#funktsii-dlia-grouping-sets} +Отложено. + ### 14.12. Функции обработки временных рядов {#funktsii-obrabotki-vremennykh-riadov} +Отложено. + Сложная задача, так как вводит новый класс функций и требует его обработку в оптимизаторе запросов. В time-series СУБД нужны функции, которые зависят от последовательности значений. Или даже от последовательности значений и их меток времени. Примеры: moving average, exponential smoothing, derivative, Holt-Winters forecast. Вычисление таких функций поддерживается в ClickHouse лишь частично. Так, ClickHouse поддерживает тип данных «массив» и позволяет реализовать эти функции как функции, принимающие массивы. Но гораздо удобнее для пользователя было бы иметь возможность применить такие функции к таблице (промежуточному результату запроса после сортировки). @@ -1089,6 +1133,8 @@ Upd. Есть pull request. Готово. ### 14.13. Применимость функций высшего порядка для кортежей и Nested {#primenimost-funktsii-vysshego-poriadka-dlia-kortezhei-i-nested} +После задачи "чтение срезов столбцов". + ### 14.14. Неявные преобразования типов констант {#neiavnye-preobrazovaniia-tipov-konstant} Сделано для операторов сравнения с константами (подавляющее большинство use cases). @@ -1180,12 +1226,14 @@ Upd. Секретного изменения в работе не будет, з ### 16.5. Функции для XML и HTML escape {#funktsii-dlia-xml-i-html-escape} -### 16.6. Функции нормализации и хэширования SQL запросов {#funktsii-normalizatsii-i-kheshirovaniia-sql-zaprosov} +### 16.6. + Функции нормализации и хэширования SQL запросов {#funktsii-normalizatsii-i-kheshirovaniia-sql-zaprosov} + +Алексей Миловидов. Сделано. ## 17. Работа с географическими данными {#rabota-s-geograficheskimi-dannymi} -### 17.1. Гео-словари для определения региона по координатам {#geo-slovari-dlia-opredeleniia-regiona-po-koordinatam} +### 17.1. + Гео-словари для определения региона по координатам {#geo-slovari-dlia-opredeleniia-regiona-po-koordinatam} [Андрей Чулков](https://github.com/achulkov2), Антон Кваша, Артур Петуховский, ВШЭ. Будет основано на коде от Арслана Урташева. @@ -1198,6 +1246,7 @@ Upd. Андрей сделал прототип интерфейса и реал Upd. Андрей сделал прототип более оптимальной структуры данных. Upd. Есть обнадёживающие результаты. Upd. В ревью. +Upd. В релизе. ### 17.2. GIS типы данных и операции {#gis-tipy-dannykh-i-operatsii} @@ -1227,6 +1276,7 @@ Upd. Есть pull request. Александр Кожихов, Максим Кузнецов. Обнаружена фундаментальная проблема в реализации, доделывает предположительно [Николай Кочетов](https://github.com/KochetovNicolai). Он может делегировать задачу кому угодно. Исправление фундаментальной проблемы - есть PR. +Фундаментальная проблема решена. ### 18.2. Агрегатные функции для статистических тестов {#agregatnye-funktsii-dlia-statisticheskikh-testov} @@ -1235,16 +1285,20 @@ Upd. Есть pull request. Предлагается реализовать в ClickHouse статистические тесты (Analysis of Variance, тесты нормальности распределения и т. п.) в виде агрегатных функций. Пример: `welchTTest(value, sample_idx)`. Сделали прототип двух тестов, есть pull request. Также есть pull request для корелляции рангов. +Upd. Помержили корелляцию рангов, но ещё не помержили сравнение t-test, u-test. ### 18.3. Инфраструктура для тренировки моделей в ClickHouse {#infrastruktura-dlia-trenirovki-modelei-v-clickhouse} В очереди. +Отложено. ## 19. Улучшение работы кластера {#uluchshenie-raboty-klastera} ### 19.1. Параллельные кворумные вставки без линеаризуемости {#parallelnye-kvorumnye-vstavki-bez-linearizuemosti} +Upd. В работе, ожидается в начале октября. + Репликация данных в ClickHouse по-умолчанию является асинхронной без выделенного мастера. Это значит, что клиент, осуществляющий вставку данных, получает успешный ответ после того, как данные попали на один сервер; репликация данных по остальным серверам осуществляется в другой момент времени. Это ненадёжно, потому что допускает потерю только что вставленных данных при потере лишь одного сервера. Для решения этой проблемы, в ClickHouse есть возможность включить «кворумную» вставку. Это значит, что клиент, осуществляющий вставку данных, получает успешный ответ после того, как данные попали на несколько (кворум) серверов. Обеспечивается линеаризуемость: клиент, получает успешный ответ после того, как данные попали на несколько реплик, *которые содержат все предыдущие данные, вставленные с кворумом* (такие реплики можно называть «синхронными»), и при запросе SELECT можно выставить настройку, разрешающую только чтение с синхронных реплик. @@ -1265,6 +1319,7 @@ Upd. Есть pull request. Upd. Алексей сделал какой-то вариант, но борется с тем, что ничего не работает. Upd. Есть pull request на начальной стадии. +Upd. Взято в работу, но непонятна перспектива, так как не ясно, подлежат ли исправлению некоторые нюансы. ### 19.3. - Подключение YT Cypress или YDB как альтернативы ZooKeeper {#podkliuchenie-yt-cypress-ili-ydb-kak-alternativy-zookeeper} @@ -1349,9 +1404,9 @@ Upd. Для DISTINCT есть pull request. [Vxider](https://github.com/Vxider), ICT Есть pull request. -### 21.6. Уменьшение числа потоков для SELECT в случае тривиального INSERT SELECT {#umenshenie-chisla-potokov-dlia-select-v-sluchae-trivialnogo-insert-select} +### 21.6. + Уменьшение числа потоков для SELECT в случае тривиального INSERT SELECT {#umenshenie-chisla-potokov-dlia-select-v-sluchae-trivialnogo-insert-select} -ucasFL, в разработке. +ucasFL, в разработке. Готово. ### 21.7. Кэш результатов запросов {#kesh-rezultatov-zaprosov} @@ -1371,11 +1426,14 @@ Upd. В обсуждении. Upd. Есть нерабочий прототип, скорее всего будет отложено. Upd. Отложено до осени. +Upd. Отложено до. ### 21.8.1. Отдельный аллокатор для кэшей с ASLR {#otdelnyi-allokator-dlia-keshei-s-aslr} В прошлом году задачу пытался сделать Данила Кутенин с помощью lfalloc из Аркадии и mimalloc из Microsoft, но оба решения не были квалифицированы для использования в продакшене. Успешная реализация задачи 21.8 отменит необходимость в этой задаче, поэтому холд. +Upd. Ещё попробовали новый tcmalloc, результаты неудовлетворительные. Пока отменено. + ### 21.9. Исправить push-down выражений с помощью Processors {#ispravit-push-down-vyrazhenii-s-pomoshchiu-processors} [Николай Кочетов](https://github.com/KochetovNicolai). Требует 2.1. @@ -1384,7 +1442,7 @@ Upd. Отложено до осени. Amos Bird. -### 21.11. Peephole оптимизации запросов {#peephole-optimizatsii-zaprosov} +### 21.11. + Peephole оптимизации запросов {#peephole-optimizatsii-zaprosov} Руслан Камалов, Михаил Малафеев, Виктор Гришанин, ВШЭ @@ -1399,8 +1457,9 @@ Amos Bird. Сделано ещё несколько оптимизаций. Upd. Все вышеперечисленные оптимизации доступны в pull requests. Upd. Из них почти все помержены, осталась одна. +Upd. Помержили всё. -### 21.12. Алгебраические оптимизации запросов {#algebraicheskie-optimizatsii-zaprosov} +### 21.12. + Алгебраические оптимизации запросов {#algebraicheskie-optimizatsii-zaprosov} Руслан Камалов, Михаил Малафеев, Виктор Гришанин, ВШЭ @@ -1415,6 +1474,7 @@ Upd. Из них почти все помержены, осталась одна Несколько оптимизаций есть в PR. Upd. Все оптимизации кроме "Обращение инъективных функций в сравнениях на равенство" есть в PR. Upd. Из них больше половины помержены, осталось ещё две. +Upd. Помержили всё. ### 21.13. Fusion агрегатных функций {#fusion-agregatnykh-funktsii} @@ -1427,6 +1487,7 @@ Constraints позволяют задать выражение, истиннос Если выражение содержит равенство, то встретив в запросе одну из частей равенства, её можно заменить на другую часть равенства, если это сделает проще чтение данных или вычисление выражения. Например, задан constraint: `URLDomain = domain(URL)`. Значит, выражение `domain(URL)` можно заменить на `URLDomain`. Upd. Возможно будет отложено на следующий год. +Отложено на следующий год. ### 21.15. Многоступенчатое чтение данных вместо PREWHERE {#mnogostupenchatoe-chtenie-dannykh-vmesto-prewhere} @@ -1442,10 +1503,11 @@ Upd. Возможно будет отложено на следующий год ### 21.18. Внутренняя параллелизация мержа больших состояний агрегатных функций {#vnutrenniaia-parallelizatsiia-merzha-bolshikh-sostoianii-agregatnykh-funktsii} -### 21.19. Оптимизация сортировки {#optimizatsiia-sortirovki} +### 21.19. + Оптимизация сортировки {#optimizatsiia-sortirovki} Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ. В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса. +Upd. Сделаны самые существенные из предложенных вариантов. \+ 1. Оптимизация top sort. @@ -1481,11 +1543,13 @@ Upd. Вместо этого будем делать задачу 1.16. ### 21.22. Userspace page cache {#userspace-page-cache} Требует 21.8. +Отложено. -### 21.23. Ускорение работы с вторичными индексами {#uskorenie-raboty-s-vtorichnymi-indeksami} +### 21.23. + Ускорение работы с вторичными индексами {#uskorenie-raboty-s-vtorichnymi-indeksami} zhang2014. Есть pull request. +Готово. ## 22. Долги и недоделанные возможности {#dolgi-i-nedodelannye-vozmozhnosti} @@ -1679,15 +1743,18 @@ Q1. [Николай Кочетов](https://github.com/KochetovNicolai). ### 24.2. Экспериментальные алгоритмы сжатия {#eksperimentalnye-algoritmy-szhatiia} +Отложено. + ClickHouse поддерживает LZ4 и ZSTD для сжатия данных. Эти алгоритмы являются парето-оптимальными по соотношению скорости и коэффициентам сжатия среди достаточно известных. Тем не менее, существуют менее известные алгоритмы сжатия, которые могут превзойти их по какому-либо критерию. Из потенциально более быстрых по сравнимом коэффициенте сжатия: Lizard, LZSSE, density. Из более сильных: bsc и csc. Необходимо изучить эти алгоритмы, добавить их поддержку в ClickHouse и исследовать их работу на тестовых датасетах. -### 24.3. Экспериментальные кодеки {#eksperimentalnye-kodeki} +### 24.3. - Экспериментальные кодеки {#eksperimentalnye-kodeki} Существуют специализированные алгоритмы кодирования числовых последовательностей: Group VarInt, MaskedVByte, PFOR. Необходимо изучить наиболее эффективные реализации этих алгоритмов. Примеры вы сможете найти на https://github.com/lemire и https://github.com/powturbo/ а также https://github.com/schizofreny/middle-out Внедрить их в ClickHouse в виде кодеков и изучить их работу на тестовых датасетах. Upd. Есть два pull requests в начальной стадии, отложено. +Upd. Отменено. ### 24.4. Шифрование в ClickHouse на уровне VFS {#shifrovanie-v-clickhouse-na-urovne-vfs} @@ -1697,6 +1764,7 @@ Upd. Есть два pull requests в начальной стадии, отло Обсуждаются детали реализации. Q3/Q4. Виталий Баранов. +Отложено, после бэкапов. ### 24.5. Поддержка функций шифрования для отдельных значений {#podderzhka-funktsii-shifrovaniia-dlia-otdelnykh-znachenii} @@ -1706,6 +1774,7 @@ Upd. Есть два pull requests в начальной стадии, отло Для этого требуется реализовать функции шифрования и расшифрования, доступные из SQL. Для шифрования реализовать возможность добавления нужного количества случайных бит для исключения одинаковых зашифрованных значений на одинаковых данных. Это позволит реализовать возможность «забывания» данных без удаления строк таблицы: можно шифровать данные разных клиентов разными ключами, и для того, чтобы забыть данные одного клиента, потребуется всего лишь удалить ключ. Делает Василий Немков, Altinity +Есть pull request в процессе ревью, исправляем проблемы производительности. ### 24.6. Userspace RAID {#userspace-raid} @@ -1722,6 +1791,7 @@ RAID позволяет одновременно увеличить надёжн Для преодоления этих ограничений, предлагается реализовать в ClickHouse встроенный алгоритм расположения данных на дисках. Есть pull request на начальной стадии. +Отложено. ### 24.7. Вероятностные структуры данных для фильтрации по подзапросам {#veroiatnostnye-struktury-dannykh-dlia-filtratsii-po-podzaprosam} @@ -1762,6 +1832,7 @@ Upd. Есть pull request. В стадии ревью. Готово. Рустам Гусейн-заде, ВШЭ. Есть pull request на промежуточной стадии. +Отложено. ### 24.11. User Defined Functions {#user-defined-functions} @@ -1785,7 +1856,7 @@ ClickHouse предоставляет достаточно богатый наб Upd. В работе два варианта реализации UDF. -### 24.12. GPU offloading {#gpu-offloading} +### 24.12. - GPU offloading {#gpu-offloading} Риск состоит в том, что даже известные GPU базы, такие как OmniSci, работают медленнее, чем ClickHouse. Преимущество возможно только на полной сортировке и JOIN. @@ -1794,10 +1865,11 @@ Upd. В работе два варианта реализации UDF. В компании nVidia сделали прототип offloading вычисления GROUP BY с некоторыми из агрегатных функций в ClickHouse и обещат предоставить исходники в публичный доступ для дальнейшего развития. Предлагается изучить этот прототип и расширить его применимость для более широкого сценария использования. В качестве альтернативы, предлагается изучить исходные коды системы `OmniSci` или `Alenka` или библиотеку `CUB` https://nvlabs.github.io/cub/ и применить некоторые из алгоритмов в ClickHouse. Upd. В компании nVidia выложили прототип, теперь нужна интеграция в систему сборки. -Upd. Интеграция в систему сборки - Иван Лежанкин. +Upd. Интеграция в систему сборки - Иван Лежанкин (не сделано). Upd. Есть прототип bitonic sort. Upd. Прототип bitonic sort помержен, но целесообразность под вопросом (он работает медленнее). Наверное надо будет подержать и удалить. +Удалили. ### 24.13. Stream запросы {#stream-zaprosy} @@ -1819,6 +1891,8 @@ Upd. Есть два прототипа от внешних контрибьют В прошлом году исследование по этой задаче сделал Егор Соловьёв, ВШЭ и Яндекс.Такси. Его исследование показало, что алгоритм нельзя существенно улучшить путём изменения параметров. Но исследование лажовое, так как рассмотрен только уже использующийся алгоритм. То есть, задача остаётся открытой. +Отложено. + ### 24.17. Экспериментальные способы ускорения параллельного GROUP BY {#eksperimentalnye-sposoby-uskoreniia-parallelnogo-group-by} Максим Серебряков @@ -1831,9 +1905,12 @@ Upd. Есть pull request - в большинстве случаев однов ### 24.19. Промежуточное состояние GROUP BY как структура данных для key-value доступа {#promezhutochnoe-sostoianie-group-by-kak-struktura-dannykh-dlia-key-value-dostupa} +Отложено. + ### 24.20. Short-circuit вычисления некоторых выражений {#short-circuit-vychisleniia-nekotorykh-vyrazhenii} Два года назад задачу попробовала сделать Анастасия Царькова, ВШЭ и Яндекс, но реализация получилась слишком неудобной и её удалили. +В обсуждении. ### 24.21. Реализация в ClickHouse протокола распределённого консенсуса {#realizatsiia-v-clickhouse-protokola-raspredelionnogo-konsensusa} @@ -1851,9 +1928,10 @@ ClickHouse также может использоваться для быстр Другая экспериментальная задача - реализация эвристик для обработки данных в неизвестном построчном текстовом формате. Детектирование CSV, TSV, JSON, детектирование разделителей и форматов значений. -### 24.23. Минимальная поддержка транзакций для множества вставок/чтений {#minimalnaia-podderzhka-tranzaktsii-dlia-mnozhestva-vstavokchtenii} +### 24.23. - Минимальная поддержка транзакций для множества вставок/чтений {#minimalnaia-podderzhka-tranzaktsii-dlia-mnozhestva-vstavokchtenii} Максим Кузнецов, ВШЭ. +Отменено. Таблицы типа MergeTree состоят из набора независимых неизменяемых «кусков» данных. При вставках данных (INSERT), формируются новые куски. При модификациях данных (слияние кусков), формируются новые куски, а старые - становятся неактивными и перестают использоваться следующими запросами. Чтение данных (SELECT) производится из снэпшота множества кусков на некоторый момент времени. Таким образом, чтения и вставки не блокируют друг друга. @@ -1863,11 +1941,12 @@ ClickHouse также может использоваться для быстр Для решения этих проблем, предлагается ввести глобальные метки времени для кусков данных (сейчас уже есть инкрементальные номера кусков, но они выделяются в рамках одной таблицы). Первым шагом сделаем эти метки времени в рамках сервера. Вторым шагом сделаем метки времени в рамках всех серверов, но неточные на основе локальных часов. Третьим шагом сделаем метки времени, выдаваемые сервисом координации. -### 24.24. Реализация алгоритмов differential privacy {#realizatsiia-algoritmov-differential-privacy} +### 24.24. - Реализация алгоритмов differential privacy {#realizatsiia-algoritmov-differential-privacy} [\#6874](https://github.com/ClickHouse/ClickHouse/issues/6874) Артём Вишняков, ВШЭ. Есть pull request. +Отменено, так как решение имеет низкую практичность. ### 24.25. Интеграция в ClickHouse функциональности обработки HTTP User Agent {#integratsiia-v-clickhouse-funktsionalnosti-obrabotki-http-user-agent} @@ -1882,6 +1961,7 @@ Upd. Есть pull request. Нужно ещё чистить код библио Александр Кожихов, ВШЭ и Яндекс.YT. Upd. Есть pull request с прототипом. +Upd. Александ Кузьменков взял задачу в работу. ### 24.27. Реализация алгоритмов min-hash, sim-hash для нечёткого поиска полудубликатов {#realizatsiia-algoritmov-min-hash-sim-hash-dlia-nechiotkogo-poiska-poludublikatov} @@ -1892,10 +1972,12 @@ ucasFL, ICT. Алгоритмы min-hash и sim-hash позволяют вычислить для текста несколько хэш-значений таких, что при небольшом изменении текста, по крайней мере один из хэшей не меняется. Вычисления можно реализовать на n-грамах и словарных шинглах. Предлагается добавить поддержку этих алгоритмов в виде функций в ClickHouse и изучить их применимость для задачи нечёткого поиска полудубликатов. Есть pull request, есть что доделывать. +Upd. Николай Кочетов взял задачу в работу. ### 24.28. Другой sketch для квантилей {#drugoi-sketch-dlia-kvantilei} Похоже на quantileTiming, но с логарифмическими корзинами. См. DDSketch. +Отложено. ### 24.29. Поддержка Arrow Flight {#podderzhka-arrow-flight} @@ -1911,6 +1993,7 @@ Amos Bird, но его решение слишком громоздкое и п ### 24.31. Кореллированные подзапросы {#korellirovannye-podzaprosy} Перепиcывание в JOIN. Не раньше 21.11, 21.12, 21.9. Низкий приоритет. +Отложено. ### 24.32. Поддержка GRPC {#podderzhka-grpc} @@ -1925,6 +2008,7 @@ Amos Bird, но его решение слишком громоздкое и п Рассматривается вариант - поддержка GRPC в ClickHouse. Здесь есть неочевидные моменты, такие как - эффективная передача массивов данных в column-oriented формате - насколько удобно будет обернуть это в GRPC. Задача в работе, есть pull request. [#10136](https://github.com/ClickHouse/ClickHouse/pull/10136) +Upd. Задачу взял в работу Виталий Баранов. ## 25. DevRel {#devrel} @@ -1970,17 +2054,18 @@ Amos Bird, но его решение слишком громоздкое и п Екатерина - организация. Upd. Проведено два онлайн митапа на русском и два на английском. -### 25.11. Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон {#mitapy-zarubezhnye-vostok-ssha-niu-iork-vozmozhno-raleigh-vozmozhno-severo-zapad-sietl-kitai-pekin-snova-vozmozhno-mitap-dlia-razrabotchikov-ili-khakaton-london} +### 25.11. + Митапы зарубежные: восток США (Нью Йорк, возможно Raleigh), возможно северо-запад (Сиэтл), Китай (Пекин снова, возможно митап для разработчиков или хакатон), Лондон {#mitapy-zarubezhnye-vostok-ssha-niu-iork-vozmozhno-raleigh-vozmozhno-severo-zapad-sietl-kitai-pekin-snova-vozmozhno-mitap-dlia-razrabotchikov-ili-khakaton-london} -[Иван Блинков](https://github.com/blinkov/) - организация. Две штуки в США запланированы. Upd. Два митапа в США и один в Европе проведены. +[Иван Блинков](https://github.com/blinkov/) - организация. Две штуки в США запланированы. Upd. Два митапа в США и один в Европе проведены. Upd. Все остальные перенесены в онлайн. ### 25.12. Статья «научная» - про устройство хранения данных и индексов или whitepaper по архитектуре. Есть вариант подать на VLDB {#statia-nauchnaia-pro-ustroistvo-khraneniia-dannykh-i-indeksov-ili-whitepaper-po-arkhitekture-est-variant-podat-na-vldb} Низкий приоритет. Алексей Миловидов. -### 25.13. Участие во всех мероприятиях Яндекса, которые связаны с разработкой бэкенда, C++ разработкой или с базами данных, возможно участие в DevRel мероприятиях {#uchastie-vo-vsekh-meropriiatiiakh-iandeksa-kotorye-sviazany-s-razrabotkoi-bekenda-c-razrabotkoi-ili-s-bazami-dannykh-vozmozhno-uchastie-v-devrel-meropriiatiiakh} +### 25.13. + Участие во всех мероприятиях Яндекса, которые связаны с разработкой бэкенда, C++ разработкой или с базами данных, возможно участие в DevRel мероприятиях {#uchastie-vo-vsekh-meropriiatiiakh-iandeksa-kotorye-sviazany-s-razrabotkoi-bekenda-c-razrabotkoi-ili-s-bazami-dannykh-vozmozhno-uchastie-v-devrel-meropriiatiiakh} -Алексей Миловидов и все подготовленные докладчики +Алексей Миловидов и все подготовленные докладчики. +Upd. Участвуем. ### 25.14. Конференции в России: все HighLoad, возможно CodeFest, DUMP или UWDC, возможно C++ Russia {#konferentsii-v-rossii-vse-highload-vozmozhno-codefest-dump-ili-uwdc-vozmozhno-c-russia} @@ -1988,6 +2073,7 @@ Amos Bird, но его решение слишком громоздкое и п Upd. Есть Saint HighLoad online. Upd. Есть C++ Russia. CodeFest, DUMP, UWDC отменились. +Upd. Добавились Highload Fwdays, Матемаркетинг. ### 25.15. Конференции зарубежные: Percona, DataOps, попытка попасть на более крупные {#konferentsii-zarubezhnye-percona-dataops-popytka-popast-na-bolee-krupnye} @@ -2009,16 +2095,18 @@ DataOps отменилась. Требуется проработать вопрос безопасности и изоляции инстансов (поднятие в контейнерах с ограничениями по сети), подключение тестовых датасетов с помощью copy-on-write файловой системы; органичения ресурсов. Есть минимальный прототип. Сделал Илья Яцишин. Этот прототип не позволяет делиться ссылками на результаты запросов. +Upd. На финальной стадии инструмент для экспериментирования с разными версиями ClickHouse. ### 25.17. Взаимодействие с ВУЗами: ВШЭ, УрФУ, ICT Beijing {#vzaimodeistvie-s-vuzami-vshe-urfu-ict-beijing} Алексей Миловидов и вся группа разработки. Благодаря Robert Hodges добавлен CMU. Upd. Взаимодействие с ВШЭ 2019/2020 успешно выполнено. +Upd. Идёт подготовка к 2020/2021. ### 25.18. - Лекция в ШАД {#lektsiia-v-shad} -Алексей Миловидов +Алексей Миловидов. ### 25.19. - Участие в курсе разработки на C++ в ШАД {#uchastie-v-kurse-razrabotki-na-c-v-shad} @@ -2029,6 +2117,8 @@ Upd. Взаимодействие с ВШЭ 2019/2020 успешно выпол Существуют мало известные специализированные СУБД, способные конкурировать с ClickHouse по скорости обработки некоторых классов запросов. Пример: `TDEngine` и `DolphinDB`, `VictoriaMetrics`, а также `Apache Doris` и `LocustDB`. Предлагается изучить и классифицировать архитектурные особенности этих систем - их особенности и преимущества. Установить эти системы, загрузить тестовые данные, изучить производительность. Проанализировать, за счёт чего достигаются преимущества. Upd. Есть поползновения с TDEngine. +Upd. Добавили OmniSci, обновили MonetDB. +Также посмотрели QuestDB и VectorSQL (они не работают). ### 25.21. Повторное награждение контрибьюторов в Китае {#povtornoe-nagrazhdenie-kontribiutorov-v-kitae} @@ -2038,6 +2128,7 @@ Upd. Ждём снятия ограничений и восстановлени [Иван Блинков](https://github.com/blinkov/) - организация. Провёл мероприятие для турецкой компании. Upd. On-site заменяется на Online. +Upd. Проведены консультации для нескольких секретных компаний. ### 25.23. Новый мерч для ClickHouse {#novyi-merch-dlia-clickhouse} From ee453959424ad24e8e2977ffb74eee03e400ff58 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 23 Sep 2020 10:27:49 +0800 Subject: [PATCH 285/390] Fix datetime comparison --- src/Functions/FunctionsComparison.h | 2 +- tests/performance/datetime_comparison.xml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests/performance/datetime_comparison.xml diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 0a3d544f9e5..436502aead4 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1213,7 +1213,7 @@ public: const bool left_is_string = isStringOrFixedString(which_left); const bool right_is_string = isStringOrFixedString(which_right); - bool date_and_datetime = (left_type != right_type) && + bool date_and_datetime = (which_left.idx != which_right.idx) && which_left.isDateOrDateTime() && which_right.isDateOrDateTime(); if (left_is_num && right_is_num && !date_and_datetime) diff --git a/tests/performance/datetime_comparison.xml b/tests/performance/datetime_comparison.xml new file mode 100644 index 00000000000..2d47ded0b1a --- /dev/null +++ b/tests/performance/datetime_comparison.xml @@ -0,0 +1,5 @@ + + SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toString(toDateTime('2020-09-30 00:00:00')) + SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toUInt32(toDateTime('2020-09-30 00:00:00')) + SELECT count() FROM numbers(1000000000) WHERE materialize(now()) > toDateTime('2020-09-30 00:00:00') + From d2427a121f321ce4978134eb09eb3b32a9bd4606 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 10:15:51 +0300 Subject: [PATCH 286/390] Update decimal_casts.xml --- tests/performance/decimal_casts.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 6c090faee77..e76a49defce 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,11 +1,11 @@ - + 10G CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory - INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(25000000) SETTINGS max_threads = 8 + INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(50000000) SETTINGS max_threads = 8 DROP TABLE IF EXISTS t SELECT toUInt32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null @@ -13,8 +13,8 @@ SELECT toInt64(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null SELECT toUInt64(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null SELECT toInt128(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null - SELECT toInt256(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null - SELECT toUInt256(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null + SELECT toInt256(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t LIMIT 10000000 FORMAT Null + SELECT toUInt256(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t LIMIT 10000000 FORMAT Null SELECT toFloat32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null SELECT toFloat64(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null From 5eaa2a08d413c1354ecf8e964518dbbfacc1a8d1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 10:28:28 +0300 Subject: [PATCH 287/390] Update report.py --- docker/test/performance-comparison/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index bd5dedfceed..ddd2590eb97 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -470,12 +470,13 @@ if args.report == 'main': 'Shortest query
(sum for all runs), s', #6 # 'Runs' #7 ] + attrs = ['' for c in columns] + attrs[7] = None text = tableStart('Test Times') text += tableHeader(columns) allowed_average_run_time = 3.75 # 60 seconds per test at 7 runs - attrs = ['' for c in columns] for r in rows: anchor = f'{currentTableAnchor()}.{r[0]}' total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers From 60bb54093b47743285ff98127841d72ed228b8f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C4=9F=C4=B1zcan=20De=C4=9Firmenci?= <62724709+ycd@users.noreply.github.com> Date: Wed, 23 Sep 2020 11:18:08 +0300 Subject: [PATCH 288/390] Added Comprehension to Improve Performance --- docs/tools/translate/translate.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/tools/translate/translate.py b/docs/tools/translate/translate.py index 6486a8cbcc7..343ab09f12a 100755 --- a/docs/tools/translate/translate.py +++ b/docs/tools/translate/translate.py @@ -49,13 +49,14 @@ def translate_impl(text, target_language=None): def translate(text, target_language=None): - result = [] - for part in re.split(curly_braces_re, text): - if part.startswith('{') and part.endswith('}'): - result.append(part) - else: - result.append(translate_impl(part, target_language=target_language)) - return ''.join(result) + return "".join( + [ + part + if part.startswith("{") and part.endswith("}") + else translate_impl(part, target_language=target_language) + for part in re.split(curly_braces_re, text) + ] + ) def translate_toc(root, lang): From 2b6f4134ebb1759e4facc91ac648f81f923b0383 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 23 Sep 2020 11:21:55 +0300 Subject: [PATCH 289/390] Parallel create/fill in perf test --- docker/test/performance-comparison/compare.sh | 6 ++-- docker/test/performance-comparison/perf.py | 31 +++++++++++-------- docker/test/performance-comparison/report.py | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 9cb337c4fd6..ddcc303da0d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -726,8 +726,8 @@ create view shortness create table inconsistent_short_marking_report engine File(TSV, 'report/unexpected-query-duration.tsv') as select - multiIf(marked_short and time > 0.1, '"short" queries must run faster than 0.02 s', - not marked_short and time < 0.02, '"normal" queries must run longer than 0.1 s', + multiIf(marked_short and time > 0.1, '\"short\" queries must run faster than 0.02 s', + not marked_short and time < 0.02, '\"normal\" queries must run longer than 0.1 s', '') problem, marked_short, time, test, query_index, query_display_name @@ -1065,7 +1065,7 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. Start in a subshell, so that # it doesn't interfere with the watchdog through `wait`. - ( get_profiles || restart && get_profiles ) ||: + ( get_profiles || { restart && get_profiles ; } ) ||: # Kill the whole process group, because somehow when the subshell is killed, # the sleep inside remains alive and orphaned. diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 79cdc8ea8d2..2e047e6fb84 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -15,6 +15,7 @@ import sys import time import traceback import xml.etree.ElementTree as et +from threading import Thread from scipy import stats def tsv_escape(s): @@ -157,8 +158,11 @@ for t in tables: print(f'skipped\t{tsv_escape(skipped_message)}') sys.exit(0) -# Run create queries -create_query_templates = [q.text for q in root.findall('create_query')] +# Run create and fill queries. We will run them simultaneously for both servers, +# to save time. +# The weird search is to keep the relative order of elements, which matters, and +# etree doesn't support the appropriate xpath query. +create_query_templates = [q.text for q in root.findall('./*') if q.tag in ('create_query', 'fill_query')] create_queries = substitute_parameters(create_query_templates) # Disallow temporary tables, because the clickhouse_driver reconnects on errors, @@ -170,18 +174,19 @@ for q in create_queries: file = sys.stderr) sys.exit(1) -for conn_index, c in enumerate(all_connections): - for q in create_queries: - c.execute(q) - print(f'create\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}') +def do_create(connection, index, queries): + for q in queries: + connection.execute(q) + print(f'create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}') -# Run fill queries -fill_query_templates = [q.text for q in root.findall('fill_query')] -fill_queries = substitute_parameters(fill_query_templates) -for conn_index, c in enumerate(all_connections): - for q in fill_queries: - c.execute(q) - print(f'fill\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}') +threads = [Thread(target = do_create, args = (connection, index, create_queries)) + for index, connection in enumerate(all_connections)] + +for t in threads: + t.start() + +for t in threads: + t.join() # Run the queries in randomized order, but preserve their indexes as specified # in the test XML. To avoid using too much time, limit the number of queries diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index ddd2590eb97..e4096041d69 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -476,7 +476,7 @@ if args.report == 'main': text = tableStart('Test Times') text += tableHeader(columns) - allowed_average_run_time = 3.75 # 60 seconds per test at 7 runs + allowed_average_run_time = 1.6 # 30 seconds per test at 7 runs for r in rows: anchor = f'{currentTableAnchor()}.{r[0]}' total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers From 25e9d11e2fb8a9c7fe41e88ad8f1a250f6d62f05 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Wed, 23 Sep 2020 13:16:59 +0300 Subject: [PATCH 290/390] Add test for fixed issue (#15164) --- tests/queries/0_stateless/01479_cross_join_9855.reference | 2 ++ tests/queries/0_stateless/01479_cross_join_9855.sql | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01479_cross_join_9855.reference create mode 100644 tests/queries/0_stateless/01479_cross_join_9855.sql diff --git a/tests/queries/0_stateless/01479_cross_join_9855.reference b/tests/queries/0_stateless/01479_cross_join_9855.reference new file mode 100644 index 00000000000..a74732eabe3 --- /dev/null +++ b/tests/queries/0_stateless/01479_cross_join_9855.reference @@ -0,0 +1,2 @@ +6 +36 diff --git a/tests/queries/0_stateless/01479_cross_join_9855.sql b/tests/queries/0_stateless/01479_cross_join_9855.sql new file mode 100644 index 00000000000..0b549619489 --- /dev/null +++ b/tests/queries/0_stateless/01479_cross_join_9855.sql @@ -0,0 +1,7 @@ +SELECT count() +FROM numbers(4) AS n1, numbers(3) AS n2 +WHERE n1.number > (select avg(n.number) from numbers(3) n); + +SELECT count() +FROM numbers(4) AS n1, numbers(3) AS n2, numbers(6) AS n3 +WHERE n1.number > (select avg(n.number) from numbers(3) n); From 626c2dad9f84c686b743fa14cf3b5b870e94bd96 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Wed, 23 Sep 2020 13:17:31 +0300 Subject: [PATCH 291/390] test for fixed issue (#15166) --- .../01481_join_with_materialized.reference | 1 + .../01481_join_with_materialized.sql | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/01481_join_with_materialized.reference create mode 100644 tests/queries/0_stateless/01481_join_with_materialized.sql diff --git a/tests/queries/0_stateless/01481_join_with_materialized.reference b/tests/queries/0_stateless/01481_join_with_materialized.reference new file mode 100644 index 00000000000..b8626c4cff2 --- /dev/null +++ b/tests/queries/0_stateless/01481_join_with_materialized.reference @@ -0,0 +1 @@ +4 diff --git a/tests/queries/0_stateless/01481_join_with_materialized.sql b/tests/queries/0_stateless/01481_join_with_materialized.sql new file mode 100644 index 00000000000..833b483dc93 --- /dev/null +++ b/tests/queries/0_stateless/01481_join_with_materialized.sql @@ -0,0 +1,21 @@ +drop table if exists t1; +drop table if exists t2; + +create table t1 +( + col UInt64, + x UInt64 MATERIALIZED col + 1 +) Engine = MergeTree order by tuple(); + +create table t2 +( + x UInt64 +) Engine = MergeTree order by tuple(); + +insert into t1 values (1),(2),(3),(4),(5); +insert into t2 values (1),(2),(3),(4),(5); + +SELECT COUNT() FROM t1 INNER JOIN t2 USING x; + +drop table t1; +drop table t2; From ad94e4ae22f8ce9f13bbd83d5e1e8a618b832234 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 23 Sep 2020 14:47:55 +0300 Subject: [PATCH 292/390] fixed default values links --- docs/tools/cmake_in_clickhouse_generator.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 066ffe08df1..51940cf5104 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -42,10 +42,7 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No if len(default) == 0: formatted_default: str = "`OFF`" elif default[0] == "$": - formatted_default: str = default[2:-1] - formatted_default: str = default_anchor_str.format( - name=formatted_default, - anchor=make_anchor(formatted_default)) + formatted_default: str = "`{}`".format(default[2:-1]) else: formatted_default: str = "`" + default + "`" @@ -91,10 +88,10 @@ def process_file(root_path: str, input_name: str) -> None: build_entity(input_name, entity, get_line_and_comment(entity[0])) def process_folder(root_path:str, name: str) -> None: - for root, _, files in os.walk(name): + for root, _, files in os.walk(os.path.join(root_path, name)): for f in files: if f == "CMakeLists.txt" or ".cmake" in f: - process_file(root_path, os.path.join(root, f)) + process_file(root, f) def generate_cmake_flags_files(root_path: str) -> None: output_file_name: str = os.path.join(root_path, "docs/en/development/cmake-in-clickhouse.md") @@ -122,11 +119,11 @@ def generate_cmake_flags_files(root_path: str) -> None: f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" + + f.write("\n### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" + table_header) for k in sorted_keys: - if k.startswith("ENABLE_") and entities[k][0].startswith("cmake"): + if k.startswith("ENABLE_") and ".cmake" in entities[k][0]: f.write(entities[k][1] + "\n") ignored_keys.append(k) From 1eb8f0a49549baa4db9de3d7fc6f4a1066fc3c58 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 23 Sep 2020 15:06:54 +0300 Subject: [PATCH 293/390] Fix bug in storage buffer during alter --- src/Storages/StorageBuffer.cpp | 8 +++++-- ...ffer_table_alter_block_structure.reference | 3 +++ ...506_buffer_table_alter_block_structure.sql | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01506_buffer_table_alter_block_structure.reference create mode 100644 tests/queries/0_stateless/01506_buffer_table_alter_block_structure.sql diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 5b9957f4ed4..cf4bd2714f1 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -447,7 +447,9 @@ private: /// Sort the columns in the block. This is necessary to make it easier to concatenate the blocks later. Block sorted_block = block.sortColumns(); - if (!buffer.data) + /// During alter we flash all buffers to storage, but buffer structure + /// may change and we update it from the new incoming block + if (buffer.data.rows() == 0) { buffer.data = sorted_block.cloneEmpty(); } @@ -829,7 +831,9 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context, checkAlterIsPossible(params, context.getSettingsRef()); auto metadata_snapshot = getInMemoryMetadataPtr(); - /// So that no blocks of the old structure remain. + /// Flush all buffers to storages, so that no non-empty blocks of the old + /// structure remain. Structure of empty blocks will be updated during first + /// insert. optimize({} /*query*/, metadata_snapshot, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); StorageInMemoryMetadata new_metadata = *metadata_snapshot; diff --git a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.reference b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.reference new file mode 100644 index 00000000000..1f90610041b --- /dev/null +++ b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.reference @@ -0,0 +1,3 @@ +2020-01-01 00:05:00 +2020-01-01 00:05:00 +2020-01-01 00:06:00 hello diff --git a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.sql b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.sql new file mode 100644 index 00000000000..cba7d84fac6 --- /dev/null +++ b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS buf_dest; +DROP TABLE IF EXISTS buf; + +CREATE TABLE buf_dest (timestamp DateTime) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (timestamp); + +CREATE TABLE buf (timestamp DateTime) Engine = Buffer(currentDatabase(), buf_dest, 16, 3, 20, 2000000, 20000000, 100000000, 300000000);; + +INSERT INTO buf (timestamp) VALUES (toDateTime('2020-01-01 00:05:00')); + +ALTER TABLE buf_dest ADD COLUMN s String; +ALTER TABLE buf ADD COLUMN s String; + +SELECT * FROM buf; + +INSERT INTO buf (timestamp, s) VALUES (toDateTime('2020-01-01 00:06:00'), 'hello'); + +SELECT * FROM buf ORDER BY timestamp; + +DROP TABLE IF EXISTS buf; +DROP TABLE IF EXISTS buf_dest; From f404925397ead1220d99d5d45de0cf054a2ba712 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 23 Sep 2020 15:19:36 +0300 Subject: [PATCH 294/390] More optimal --- src/Storages/StorageBuffer.cpp | 14 +++++++------- src/Storages/StorageBuffer.h | 8 +++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index cf4bd2714f1..14f188275e5 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -447,9 +447,7 @@ private: /// Sort the columns in the block. This is necessary to make it easier to concatenate the blocks later. Block sorted_block = block.sortColumns(); - /// During alter we flash all buffers to storage, but buffer structure - /// may change and we update it from the new incoming block - if (buffer.data.rows() == 0) + if (!buffer.data) { buffer.data = sorted_block.cloneEmpty(); } @@ -549,7 +547,7 @@ bool StorageBuffer::optimize( if (deduplicate) throw Exception("DEDUPLICATE cannot be specified when optimizing table of type Buffer", ErrorCodes::NOT_IMPLEMENTED); - flushAllBuffers(false); + flushAllBuffers(false, true); return true; } @@ -597,14 +595,14 @@ bool StorageBuffer::checkThresholdsImpl(size_t rows, size_t bytes, time_t time_p } -void StorageBuffer::flushAllBuffers(const bool check_thresholds) +void StorageBuffer::flushAllBuffers(bool check_thresholds, bool reset_blocks_structure) { for (auto & buf : buffers) - flushBuffer(buf, check_thresholds); + flushBuffer(buf, check_thresholds, false, reset_blocks_structure); } -void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked) +void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked, bool reset_block_structure) { Block block_to_write; time_t current_time = time(nullptr); @@ -657,6 +655,8 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc try { writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id, global_context)); + if (reset_block_structure) + buffer.data.clear(); } catch (...) { diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 8f1354399ef..b18b574ec6c 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -130,9 +130,11 @@ private: Poco::Logger * log; - void flushAllBuffers(bool check_thresholds = true); - /// Reset the buffer. If check_thresholds is set - resets only if thresholds are exceeded. - void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false); + void flushAllBuffers(bool check_thresholds = true, bool reset_blocks_structure = false); + /// Reset the buffer. If check_thresholds is set - resets only if thresholds + /// are exceeded. If reset_block_structure is set - clears inner block + /// structure inside buffer (useful in OPTIMIZE and ALTER). + void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false, bool reset_block_structure = false); bool checkThresholds(const Buffer & buffer, time_t current_time, size_t additional_rows = 0, size_t additional_bytes = 0) const; bool checkThresholdsImpl(size_t rows, size_t bytes, time_t time_passed) const; From 59cc997d9c34548bee098fc775ab45c36a21885a Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 23 Sep 2020 15:12:13 +0200 Subject: [PATCH 295/390] Fix documentation for map functions --- .../functions/tuple-map-functions.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index f826b810d23..55f34b5831e 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -9,8 +9,7 @@ toc_title: Working with maps Collect all the keys and sum corresponding values. -Arguments are tuples of two arrays, where items in the first array represent keys, and the second array -contains values for the each key. +Arguments are tuples of two arrays, where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promotable to the one type (Int64, UInt64 or Float64). The common promoted type is used as a type for the result array. @@ -30,8 +29,7 @@ SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTy Collect all the keys and subtract corresponding values. -Arguments are tuples of two arrays, where items in the first array represent keys, and the second array -contains values for the each key. +Arguments are tuples of two arrays, where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promotable to the one type (Int64, UInt64 or Float64). The common promoted type is used as a type for the result array. @@ -45,25 +43,24 @@ SELECT mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt3 ┌─res────────────┬─type──────────────────────────────┐ │ ([1,2],[-1,0]) │ Tuple(Array(UInt8), Array(Int64)) │ └────────────────┴───────────────────────────────────┘ -```` +``` ## mapPopulateSeries {#function-mappopulateseries} Syntax: `mapPopulateSeries((keys : Array(), values : Array()[, max : ])` -Generates a map, where keys are a series of numbers, from minimum to maximum keys (or `max` argument if it specified) taken from `keys` array with step size of one, -and corresponding values taken from `values` array. If the value is not specified for the key, then it uses default value in the resulting map. +Generates a map, where keys are a series of numbers, from minimum to maximum keys (or `max` argument if it specified) taken from `keys` array with step size of one, and corresponding values taken from `values` array. If the value is not specified for the key, then it uses default value in the resulting map. For repeated keys only the first value (in order of appearing) gets associated with the key. The number of elements in `keys` and `values` must be the same for each row. Returns a tuple of two arrays: keys in sorted order, and values the corresponding keys. -``` sql +```sql select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type; ``` -``` text +```text ┌─res──────────────────────────┬─type──────────────────────────────┐ │ ([1,2,3,4,5],[11,22,0,44,0]) │ Tuple(Array(UInt8), Array(UInt8)) │ └──────────────────────────────┴───────────────────────────────────┘ From fef9e0d06d5b6a850b876a90180d097e59c1a528 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 23 Sep 2020 16:53:54 +0300 Subject: [PATCH 296/390] fixed the invalid AMP generation --- docs/tools/cmake_in_clickhouse_generator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 51940cf5104..62672532cdf 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -39,6 +39,10 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No if name in entities: return + # cannot escape the { in macro option description -> invalid AMP html + if name == "USE_INTERNAL_${LIB_NAME_UC}_LIBRARY ": + return + if len(default) == 0: formatted_default: str = "`OFF`" elif default[0] == "$": @@ -85,7 +89,7 @@ def process_file(root_path: str, input_name: str) -> None: if matches: for entity in matches: - build_entity(input_name, entity, get_line_and_comment(entity[0])) + build_entity(os.path.join(root_path, input_name), entity, get_line_and_comment(entity[0])) def process_folder(root_path:str, name: str) -> None: for root, _, files in os.walk(os.path.join(root_path, name)): From af50f9d52dd4c12ea4c641940a40563faf69dac0 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 23 Sep 2020 16:58:16 +0300 Subject: [PATCH 297/390] another fix --- docs/tools/cmake_in_clickhouse_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 62672532cdf..81851098dc8 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -40,7 +40,8 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No return # cannot escape the { in macro option description -> invalid AMP html - if name == "USE_INTERNAL_${LIB_NAME_UC}_LIBRARY ": + # Skipping "USE_INTERNAL_${LIB_NAME_UC}_LIBRARY" + if "LIB_NAME_UC" in name: return if len(default) == 0: From df5db10e016563d5508de2585db76b711660e118 Mon Sep 17 00:00:00 2001 From: myrrc Date: Wed, 23 Sep 2020 17:09:34 +0300 Subject: [PATCH 298/390] another better-or-worse fix --- docs/tools/cmake_in_clickhouse_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index 81851098dc8..bc58a68c409 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -90,7 +90,7 @@ def process_file(root_path: str, input_name: str) -> None: if matches: for entity in matches: - build_entity(os.path.join(root_path, input_name), entity, get_line_and_comment(entity[0])) + build_entity(os.path.join(root_path[6:], input_name), entity, get_line_and_comment(entity[0])) def process_folder(root_path:str, name: str) -> None: for root, _, files in os.walk(os.path.join(root_path, name)): From 43571bf8d6ee146c0d546b927eecbde30fb63ec1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 23 Sep 2020 18:18:03 +0300 Subject: [PATCH 299/390] Fix script --- docs/tools/cmake_in_clickhouse_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index bc58a68c409..b15df76151e 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -132,14 +132,14 @@ def generate_cmake_flags_files(root_path: str) -> None: f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("### External libraries system/bundled mode\n" + table_header) + f.write("\n### External libraries system/bundled mode\n" + table_header) for k in sorted_keys: if k.startswith("USE_INTERNAL_"): f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("### Other flags\n" + table_header) + f.write("\n### Other flags\n" + table_header) for k in sorted(set(sorted_keys).difference(set(ignored_keys))): f.write(entities[k][1] + "\n") From 9f137e77e781c7f1a571fafe7ba293eed1ce6077 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 18:41:52 +0300 Subject: [PATCH 300/390] Update decimal_casts.xml --- tests/performance/decimal_casts.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index e76a49defce..7f609c8bd9d 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,11 +1,11 @@ - 10G + 15G CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory - INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(50000000) SETTINGS max_threads = 8 + INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 8 DROP TABLE IF EXISTS t SELECT toUInt32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null From 74e785a2fedb8f6c49d85e9e66ae7c34bf29195a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 23 Sep 2020 20:42:46 +0300 Subject: [PATCH 301/390] Add optimize setting to explain plan. --- src/Interpreters/InterpreterExplainQuery.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index c936556ce39..a0a63dfed08 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -119,13 +119,17 @@ struct QueryPlanSettings { QueryPlan::ExplainPlanOptions query_plan_options; + /// Apply query plan optimisations. + bool optimize = true; + constexpr static char name[] = "PLAN"; std::unordered_map> boolean_settings = { {"header", query_plan_options.header}, {"description", query_plan_options.description}, - {"actions", query_plan_options.actions} + {"actions", query_plan_options.actions}, + {"optimize", optimize}, }; }; @@ -248,7 +252,8 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() InterpreterSelectWithUnionQuery interpreter(ast.getExplainedQuery(), context, SelectQueryOptions()); interpreter.buildQueryPlan(plan); - plan.optimize(); + if (settings.optimize) + plan.optimize(); WriteBufferFromOStream buffer(ss); plan.explainPlan(buffer, settings.query_plan_options); From a8266a3d3ca99c6c4dc22d219a293deccc759609 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 23 Sep 2020 21:28:59 +0300 Subject: [PATCH 302/390] fix DDDL with cross-replication and Atomic --- programs/copier/ClusterCopier.cpp | 4 +- src/Common/Macros.cpp | 54 +++++++++++++------ src/Common/Macros.h | 21 ++++++-- src/Interpreters/Cluster.cpp | 17 ++++++ src/Interpreters/Cluster.h | 4 ++ src/Interpreters/InterpreterCreateQuery.cpp | 54 +++++++++++++++++-- src/Interpreters/InterpreterCreateQuery.h | 2 + .../test_cluster_copier/task0_description.xml | 4 +- .../task_month_to_week_description.xml | 4 +- .../task_test_block_size.xml | 4 +- tests/integration/test_cluster_copier/test.py | 21 ++++---- .../test_cluster_copier/trivial_test.py | 2 +- .../integration/test_distributed_ddl/test.py | 15 +++++- .../test.py | 27 ++++++++++ 14 files changed, 192 insertions(+), 41 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index b3d1ca7bcec..4ee14b14119 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -1477,7 +1477,9 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( { auto create_query_push_ast = rewriteCreateQueryStorage(task_shard.current_pull_table_create_query, task_table.table_push, task_table.engine_push_ast); - create_query_push_ast->as().if_not_exists = true; + auto & create = create_query_push_ast->as(); + create.if_not_exists = true; + InterpreterCreateQuery::prepareOnClusterQuery(create, context, task_table.cluster_push_name); String query = queryToString(create_query_push_ast); LOG_DEBUG(log, "Create destination tables. Query: {}", query); diff --git a/src/Common/Macros.cpp b/src/Common/Macros.cpp index a4981fa5be3..e3735c44359 100644 --- a/src/Common/Macros.cpp +++ b/src/Common/Macros.cpp @@ -23,18 +23,15 @@ Macros::Macros(const Poco::Util::AbstractConfiguration & config, const String & } String Macros::expand(const String & s, - size_t level, - const String & database_name, - const String & table_name, - const UUID & uuid) const + MacroExpansionInfo & info) const { if (s.find('{') == String::npos) return s; - if (level && s.size() > 65536) + if (info.level && s.size() > 65536) throw Exception("Too long string while expanding macros", ErrorCodes::SYNTAX_ERROR); - if (level >= 10) + if (info.level >= 10) throw Exception("Too deep recursion while expanding macros: '" + s + "'", ErrorCodes::SYNTAX_ERROR); String res; @@ -64,17 +61,28 @@ String Macros::expand(const String & s, /// Prefer explicit macros over implicit. if (it != macros.end()) res += it->second; - else if (macro_name == "database" && !database_name.empty()) - res += database_name; - else if (macro_name == "table" && !table_name.empty()) - res += table_name; + else if (macro_name == "database" && !info.database_name.empty()) + res += info.database_name; + else if (macro_name == "table" && !info.table_name.empty()) + res += info.table_name; else if (macro_name == "uuid") { - if (uuid == UUIDHelpers::Nil) + if (info.uuid == UUIDHelpers::Nil) throw Exception("Macro 'uuid' and empty arguments of ReplicatedMergeTree " "are supported only for ON CLUSTER queries with Atomic database engine", ErrorCodes::SYNTAX_ERROR); - res += toString(uuid); + /// For ON CLUSTER queries we don't want to require all macros definitions in initiator's config. + /// However, initiator must check that for cross-replication cluster zookeeper_path does not contain {uuid} macro. + /// It becomes impossible to check if {uuid} is contained inside some unknown macro. + if (info.level) + throw Exception("Macro 'uuid' should not be inside another macro", ErrorCodes::SYNTAX_ERROR); + res += toString(info.uuid); + info.expanded_uuid = true; + } + else if (info.ignore_unknown) + { + res += macro_name; + info.has_unknown = true; } else throw Exception("No macro '" + macro_name + @@ -84,7 +92,8 @@ String Macros::expand(const String & s, pos = end + 1; } - return expand(res, level + 1, database_name, table_name); + ++info.level; + return expand(res, info); } String Macros::getValue(const String & key) const @@ -94,9 +103,20 @@ String Macros::getValue(const String & key) const throw Exception("No macro " + key + " in config", ErrorCodes::SYNTAX_ERROR); } + +String Macros::expand(const String & s) const +{ + MacroExpansionInfo info; + return expand(s, info); +} + String Macros::expand(const String & s, const StorageID & table_id, bool allow_uuid) const { - return expand(s, 0, table_id.database_name, table_id.table_name, allow_uuid ? table_id.uuid : UUIDHelpers::Nil); + MacroExpansionInfo info; + info.database_name = table_id.database_name; + info.table_name = table_id.table_name; + info.uuid = allow_uuid ? table_id.uuid : UUIDHelpers::Nil; + return expand(s, info); } Names Macros::expand(const Names & source_names, size_t level) const @@ -104,8 +124,12 @@ Names Macros::expand(const Names & source_names, size_t level) const Names result_names; result_names.reserve(source_names.size()); + MacroExpansionInfo info; for (const String & name : source_names) - result_names.push_back(expand(name, level)); + { + info.level = level; + result_names.push_back(expand(name, info)); + } return result_names; } diff --git a/src/Common/Macros.h b/src/Common/Macros.h index bcd6075782e..6e4f25d55ef 100644 --- a/src/Common/Macros.h +++ b/src/Common/Macros.h @@ -27,15 +27,28 @@ public: Macros() = default; Macros(const Poco::Util::AbstractConfiguration & config, const String & key); + struct MacroExpansionInfo + { + /// Settings + String database_name; + String table_name; + UUID uuid = UUIDHelpers::Nil; + bool ignore_unknown = false; + + /// Information about macro expansion + size_t level = 0; + bool expanded_uuid = false; + bool has_unknown = false; + }; + /** Replace the substring of the form {macro_name} with the value for macro_name, obtained from the config file. * If {database} and {table} macros aren`t defined explicitly, expand them as database_name and table_name respectively. * level - the level of recursion. */ String expand(const String & s, - size_t level = 0, - const String & database_name = "", - const String & table_name = "", - const UUID & uuid = UUIDHelpers::Nil) const; + MacroExpansionInfo & info) const; + + String expand(const String & s) const; String expand(const String & s, const StorageID & table_id, bool allow_uuid) const; diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index b385e74adc5..8a98e8282a6 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -623,4 +623,21 @@ const std::string & Cluster::ShardInfo::pathForInsert(bool prefer_localhost_repl return dir_name_for_internal_replication_with_local; } +bool Cluster::maybeCrossReplication() const +{ + /// Cluster can be used for cross-replication if some replicas have different default database names, + /// so one clickhouse-server instance can contain multiple replicas. + + if (addresses_with_failover.empty()) + return false; + + const String & database_name = addresses_with_failover.front().front().default_database; + for (const auto & shard : addresses_with_failover) + for (const auto & replica : shard) + if (replica.default_database != database_name) + return true; + + return false; +} + } diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index 4985c70e6e2..c8225c81453 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -193,6 +193,10 @@ public: /// Get a new Cluster that contains all servers (all shards with all replicas) from existing cluster as independent shards. std::unique_ptr getClusterWithReplicasAsShards(const Settings & settings) const; + /// Returns false if cluster configuration doesn't allow to use it for cross-replication. + /// NOTE: true does not mean, that it's actually a cross-replication cluster. + bool maybeCrossReplication() const; + private: using SlotToShard = std::vector; SlotToShard slot_to_shard; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index cc39d6fc8a3..6f318b3658a 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -853,15 +854,60 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) return {}; } +void InterpreterCreateQuery::prepareOnClusterQuery(ASTCreateQuery & create, const Context & context, const String & cluster_name) +{ + if (create.attach) + return; + + /// For CREATE query generate UUID on initiator, so it will be the same on all hosts. + /// It will be ignored if database does not support UUIDs. + if (create.uuid == UUIDHelpers::Nil) + create.uuid = UUIDHelpers::generateV4(); + + /// For cross-replication cluster we cannot use UUID in replica path. + String cluster_name_expanded = context.getMacros()->expand(cluster_name); + ClusterPtr cluster = context.getCluster(cluster_name_expanded); + + if (cluster->maybeCrossReplication()) + { + /// Check that {uuid} macro is not used in zookeeper_path for ReplicatedMergeTree. + /// Otherwise replicas will generate different paths. + if (!create.storage) + return; + if (!create.storage->engine) + return; + if (!startsWith(create.storage->engine->name, "Replicated")) + return; + + bool has_explicit_zk_path_arg = create.storage->engine->arguments && + create.storage->engine->arguments->children.size() >= 2 && + create.storage->engine->arguments->children[0]->as() && + create.storage->engine->arguments->children[0]->as()->value.getType() == Field::Types::String; + + if (has_explicit_zk_path_arg) + { + String zk_path = create.storage->engine->arguments->children[0]->as()->value.get(); + Macros::MacroExpansionInfo info; + info.uuid = create.uuid; + info.ignore_unknown = true; + context.getMacros()->expand(zk_path, info); + if (!info.expanded_uuid) + return; + } + + throw Exception("Seems like cluster is configured for cross-replication, " + "but zookeeper_path for ReplicatedMergeTree is not specified or contains {uuid} macro. " + "It's not supported for cross replication, because tables must have different UUIDs. " + "Please specify unique zookeeper_path explicitly.", ErrorCodes::INCORRECT_QUERY); + } +} + BlockIO InterpreterCreateQuery::execute() { auto & create = query_ptr->as(); if (!create.cluster.empty()) { - /// For CREATE query generate UUID on initiator, so it will be the same on all hosts. - /// It will be ignored if database does not support UUIDs. - if (!create.attach && create.uuid == UUIDHelpers::Nil) - create.uuid = UUIDHelpers::generateV4(); + prepareOnClusterQuery(create, context, create.cluster); return executeDDLQueryOnCluster(query_ptr, context, getRequiredAccess()); } diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 4a5d57c11d1..07fca5f3910 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -55,6 +55,8 @@ public: static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context, bool sanity_check_compression_codecs); static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints); + static void prepareOnClusterQuery(ASTCreateQuery & create, const Context & context, const String & cluster_name); + private: struct TableProperties { diff --git a/tests/integration/test_cluster_copier/task0_description.xml b/tests/integration/test_cluster_copier/task0_description.xml index 72eff8d464d..d56053ffd39 100644 --- a/tests/integration/test_cluster_copier/task0_description.xml +++ b/tests/integration/test_cluster_copier/task0_description.xml @@ -33,7 +33,7 @@ 3 4 5 6 1 2 0 - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/hits', '{replica}') PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 + ENGINE=ReplicatedMergeTree PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16 d + 1 @@ -93,4 +93,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/task_month_to_week_description.xml b/tests/integration/test_cluster_copier/task_month_to_week_description.xml index ee134603310..26dfc7d3e00 100644 --- a/tests/integration/test_cluster_copier/task_month_to_week_description.xml +++ b/tests/integration/test_cluster_copier/task_month_to_week_description.xml @@ -34,7 +34,7 @@ ENGINE= - ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/b', '{replica}') + ReplicatedMergeTree PARTITION BY toMonday(date) ORDER BY d @@ -97,4 +97,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/task_test_block_size.xml b/tests/integration/test_cluster_copier/task_test_block_size.xml index ea63d580c1c..c9c99a083ea 100644 --- a/tests/integration/test_cluster_copier/task_test_block_size.xml +++ b/tests/integration/test_cluster_copier/task_test_block_size.xml @@ -28,7 +28,7 @@ ENGINE= - ReplicatedMergeTree('/clickhouse/tables/cluster{cluster}/{shard}/test_block_size', '{replica}') + ReplicatedMergeTree ORDER BY d PARTITION BY partition @@ -99,4 +99,4 @@ - \ No newline at end of file + diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 2a9e696ca46..88dac06f158 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -81,11 +81,11 @@ class Task1: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( cluster_num)) ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + + "ENGINE=ReplicatedMergeTree " + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") @@ -110,10 +110,11 @@ class Task1: class Task2: - def __init__(self, cluster): + def __init__(self, cluster, unique_zk_path): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition" self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read() + self.unique_zk_path = unique_zk_path def start(self): instance = cluster.instances['s0_0_0'] @@ -121,11 +122,13 @@ class Task2: for cluster_num in ["0", "1"]: ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( cluster_num)) ddl_check_query(instance, - "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") + "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/" + self.unique_zk_path + "', " + "'{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") @@ -169,7 +172,7 @@ class Task_test_block_size: ddl_check_query(instance, """ CREATE TABLE test_block_size ON CLUSTER shard_0_0 (partition Date, d UInt64) - ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') + ENGINE=ReplicatedMergeTree ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) instance.query( @@ -332,17 +335,17 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs @pytest.mark.timeout(600) def test_copy_month_to_week_partition(started_cluster): - execute_task(Task2(started_cluster), []) + execute_task(Task2(started_cluster, "test1"), []) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering(started_cluster): - execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster, "test2"), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): - execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + execute_task(Task2(started_cluster, "test3"), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) def test_block_size(started_cluster): diff --git a/tests/integration/test_cluster_copier/trivial_test.py b/tests/integration/test_cluster_copier/trivial_test.py index 3d0c5d0f5b0..035faf0bb9f 100644 --- a/tests/integration/test_cluster_copier/trivial_test.py +++ b/tests/integration/test_cluster_copier/trivial_test.py @@ -59,7 +59,7 @@ class TaskTrivial: for node in [source, destination]: node.query("DROP DATABASE IF EXISTS default") - node.query("CREATE DATABASE IF NOT EXISTS default ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS default") source.query("CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1) " "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 7f254d03015..b788dafe167 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -327,6 +327,8 @@ def test_replicated_without_arguments(test_cluster): rules = test_cluster.pm_random_drops.pop_rules() instance = test_cluster.instances['ch1'] test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic") + assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ + instance.query_and_get_error("CREATE TABLE test_atomic.rmt (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster") @@ -334,9 +336,20 @@ def test_replicated_without_arguments(test_cluster): "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY n") test_cluster.ddl_check_query(instance, "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") + assert instance.query("SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'databases') WHERE uuid != 0 AND name='test_atomic'") == "1\n" + assert instance.query("SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'tables') WHERE uuid != 0 AND name='rmt'") == "1\n" + test_cluster.ddl_check_query(instance, "DROP DATABASE test_atomic ON CLUSTER cluster") + + test_cluster.ddl_check_query(instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary") + assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ + instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") + assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ + instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") + test_cluster.ddl_check_query(instance, "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n") + test_cluster.ddl_check_query(instance, "DROP DATABASE test_ordinary ON CLUSTER cluster") test_cluster.pm_random_drops.push_rules(rules) diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/test.py b/tests/integration/test_distributed_ddl_on_cross_replication/test.py index 16238f0326d..85800b2e5e6 100644 --- a/tests/integration/test_distributed_ddl_on_cross_replication/test.py +++ b/tests/integration/test_distributed_ddl_on_cross_replication/test.py @@ -77,3 +77,30 @@ def test_alter_ddl(started_cluster): node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0') + +def test_atomic_database(started_cluster): + node1.query('''DROP DATABASE IF EXISTS replica_1 ON CLUSTER cross_3shards_2replicas; + DROP DATABASE IF EXISTS replica_2 ON CLUSTER cross_3shards_2replicas; + CREATE DATABASE replica_1 ON CLUSTER cross_3shards_2replicas ENGINE=Atomic; + CREATE DATABASE replica_2 ON CLUSTER cross_3shards_2replicas ENGINE=Atomic;''') + + assert "It's not supported for cross replication" in \ + node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") + assert "It's not supported for cross replication" in \ + node1.query_and_get_error("CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") + assert "It's not supported for cross replication" in \ + node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") + assert "It's not supported for cross replication" in \ + node1.query_and_get_error("CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") + assert "For a distributed DDL on circular replicated cluster its table name must be qualified by database name" in \ + node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n") + + node1.query("CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n") + node1.query("CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard_bk}/rmt/', '{replica_bk}') ORDER BY n") + + assert node1.query("SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'databases') WHERE uuid != 0 AND name='replica_1'") == "1\n" + assert node1.query("SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'tables') WHERE uuid != 0 AND name='rmt'") == "2\n" + + node1.query("INSERT INTO replica_1.rmt VALUES (1, 'test')") + node2.query("SYSTEM SYNC REPLICA replica_2.rmt", timeout=5) + assert_eq_with_retry(node2, "SELECT * FROM replica_2.rmt", '1\ttest') From 2d4bdcfec0e2982f102a72b6f7fd40acd197e421 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 23 Sep 2020 22:31:47 +0300 Subject: [PATCH 303/390] file and executables --- tests/integration/helpers/external_sources.py | 6 +++--- .../test_executable_cache.py | 10 +++------- .../test_executable_hashed.py | 8 ++++---- .../test_file.py | 6 +++--- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index 0d01a1bcbfd..a52cf7a02d8 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -333,16 +333,16 @@ class _SourceExecutableBase(ExternalSource): user='root') -class SourceExecutableCache(_SourceExecutableBase): +class SourceExecutableHashed(_SourceExecutableBase): def _get_cmd(self, path): return "cat {}".format(path) def compatible_with_layout(self, layout): - return 'cache' not in layout.name + return 'hashed' in layout.name -class SourceExecutableHashed(_SourceExecutableBase): +class SourceExecutableCache(_SourceExecutableBase): def _get_cmd(self, path): return "cat - >/dev/null;cat {}".format(path) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py index 87b73d3acc1..1d741d5271c 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py @@ -46,7 +46,7 @@ def setup_module(module): for fname in os.listdir(DICT_CONFIG_PATH): dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): @@ -69,14 +69,10 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +@pytest.mark.parametrize("layout_name", ['cache']) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +@pytest.mark.parametrize("layout_name", ['complex_key_cache']) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) - -@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) -def test_ranged(started_cluster, layout_name): - ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py index 5ab5ee545e7..03af42bb1d4 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py @@ -46,7 +46,7 @@ def setup_module(module): for fname in os.listdir(DICT_CONFIG_PATH): dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): @@ -69,14 +69,14 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE) +@pytest.mark.parametrize("layout_name", ['hashed']) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX) +@pytest.mark.parametrize("layout_name", ['complex_key_hashed']) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) - + @pytest.mark.parametrize("layout_name", LAYOUTS_RANGED) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py index 893d095a143..f786bda847f 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py @@ -46,7 +46,7 @@ def setup_module(module): for fname in os.listdir(DICT_CONFIG_PATH): dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): @@ -69,11 +69,11 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("cache, direct"))) ) +@pytest.mark.parametrize("layout_name", set(LAYOUTS_SIMPLE).difference({'cache', 'direct'}) ) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("complex_key_cache, complex_key_direct")))) +@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_COMPLEX).difference({'complex_key_cache', 'complex_key_direct'}))) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) From c64ac8596c13eb9b2e9555531c1e89407820a26d Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 23 Sep 2020 22:36:47 +0300 Subject: [PATCH 304/390] better --- .../test_clickhouse_local.py | 2 +- .../test_clickhouse_remote.py | 2 +- .../test_dictionaries_all_layouts_separate_sources/test_http.py | 2 +- .../test_https.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py index c8597708c04..1adc02ba6aa 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py @@ -46,7 +46,7 @@ def setup_module(module): for fname in os.listdir(DICT_CONFIG_PATH): dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py index 4f190e4c332..4e7f307b959 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py @@ -48,7 +48,7 @@ def setup_module(module): cluster.add_instance('clickhouse1', main_configs=main_configs) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py index 8c7519fd0a7..80baee5ee45 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py @@ -48,7 +48,7 @@ def setup_module(module): cluster.add_instance('clickhouse1', main_configs=main_configs) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py index c5280aa0076..ccac2cfd268 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py @@ -48,7 +48,7 @@ def setup_module(module): cluster.add_instance('clickhouse1', main_configs=main_configs) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries) def teardown_module(module): From b2d66770ee80d1c249ac2726b6820b86616ee558 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 23 Sep 2020 21:35:29 +0300 Subject: [PATCH 305/390] Update protobuf to the latest version. --- contrib/protobuf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/protobuf b/contrib/protobuf index d6a10dd3db5..445d1ae73a4 160000 --- a/contrib/protobuf +++ b/contrib/protobuf @@ -1 +1 @@ -Subproject commit d6a10dd3db55d8f7f9e464db9151874cde1f79ec +Subproject commit 445d1ae73a450b1e94622e7040989aa2048402e3 From ba3257de22ab6ccb657c5d10c75698a82306fbb9 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 23 Sep 2020 23:20:59 +0300 Subject: [PATCH 306/390] Disable warnings in protobuf's headers. --- contrib/protobuf-cmake/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt index 683429194fc..1f8d9b02b3e 100644 --- a/contrib/protobuf-cmake/CMakeLists.txt +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -11,3 +11,7 @@ else () endif () add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}") + +# We don't want to stop compilation on warnings in protobuf's headers. +# The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake +set_property(TARGET libprotobuf PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${protobuf_SOURCE_DIR}/src) From 179807b0b405c2a02155b4e6cc5b06c881bc4ae6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 23 Sep 2020 23:45:39 +0300 Subject: [PATCH 307/390] performance comparison --- docker/test/performance-comparison/report.py | 51 +++++++++++--------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index ddd2590eb97..c5cfd3bfff9 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -187,8 +187,10 @@ def td(value, cell_attributes = ''): cell_attributes = cell_attributes, value = value) -def th(x): - return '' +def th(value, cell_attributes = ''): + return ''.format( + cell_attributes = cell_attributes, + value = value) def tableRow(cell_values, cell_attributes = [], anchor=None): return tr( @@ -199,8 +201,13 @@ def tableRow(cell_values, cell_attributes = [], anchor=None): if a is not None and v is not None]), anchor) -def tableHeader(r): - return tr(''.join([th(f) for f in r])) +def tableHeader(cell_values, cell_attributes = []): + return tr( + ''.join([th(v, a) + for v, a in itertools.zip_longest( + cell_values, cell_attributes, + fillvalue = '') + if a is not None and v is not None])) def tableStart(title): cls = '-'.join(title.lower().split(' ')[:3]); @@ -377,16 +384,16 @@ if args.report == 'main': 'Ratio of speedup (-) or slowdown (+)', # 2 'Relative difference (new − old) / old', # 3 'p < 0.01 threshold', # 4 - # Failed # 5 + '', # Failed # 5 'Test', # 6 '#', # 7 'Query', # 8 ] - - text += tableHeader(columns) - attrs = ['' for c in columns] attrs[5] = None + + text += tableHeader(columns, attrs) + for row in rows: anchor = f'{currentTableAnchor()}.{row[6]}.{row[7]}' if int(row[5]): @@ -421,17 +428,17 @@ if args.report == 'main': 'New, s', #1 'Relative difference (new - old)/old', #2 'p < 0.01 threshold', #3 - # Failed #4 + '', # Failed #4 'Test', #5 '#', #6 'Query' #7 ] - - text = tableStart('Unstable Queries') - text += tableHeader(columns) - attrs = ['' for c in columns] attrs[4] = None + + text = tableStart('Unstable Queries') + text += tableHeader(columns, attrs) + for r in unstable_rows: anchor = f'{currentTableAnchor()}.{r[5]}.{r[6]}' if int(r[4]): @@ -464,17 +471,17 @@ if args.report == 'main': 'Test', #0 'Wall clock time, s', #1 'Total client time, s', #2 - 'Total queries', #3 + 'Total queries', #3 'Longest query
(sum for all runs), s', #4 'Avg wall clock time
(sum for all runs), s', #5 'Shortest query
(sum for all runs), s', #6 - # 'Runs' #7 + '', # Runs #7 ] attrs = ['' for c in columns] attrs[7] = None text = tableStart('Test Times') - text += tableHeader(columns) + text += tableHeader(columns, attrs) allowed_average_run_time = 3.75 # 60 seconds per test at 7 runs for r in rows: @@ -581,8 +588,8 @@ elif args.report == 'all-queries': return columns = [ - # Changed #0 - # Unstable #1 + '', # Changed #0 + '', # Unstable #1 'Old, s', #2 'New, s', #3 'Ratio of speedup (-) or slowdown (+)', #4 @@ -592,13 +599,13 @@ elif args.report == 'all-queries': '#', #8 'Query', #9 ] - - text = tableStart('All Query Times') - text += tableHeader(columns) - attrs = ['' for c in columns] attrs[0] = None attrs[1] = None + + text = tableStart('All Query Times') + text += tableHeader(columns, attrs) + for r in rows: anchor = f'{currentTableAnchor()}.{r[7]}.{r[8]}' if int(r[1]): From 5964e242da081443f0fad942787560751522ad67 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 24 Sep 2020 00:48:54 +0300 Subject: [PATCH 308/390] trigger CI From a85bb16c66f0c795290cf8dc740c32c4d3ba0493 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 10:33:22 +0300 Subject: [PATCH 309/390] Remove zookeeper from CI images --- docker/test/stateless/Dockerfile | 4 +--- docker/test/stateless/run.sh | 2 -- docker/test/stateless_unbundled/Dockerfile | 2 -- docker/test/stateless_unbundled/run.sh | 2 -- docker/test/stateless_with_coverage/Dockerfile | 2 -- docker/test/stateless_with_coverage/run.sh | 3 --- 6 files changed, 1 insertion(+), 14 deletions(-) diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 409a1b07bef..516d8d5842b 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -21,9 +21,7 @@ RUN apt-get update -y \ telnet \ tree \ unixodbc \ - wget \ - zookeeper \ - zookeeperd + wget RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index b6b48cd0943..adec0f51490 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -48,8 +48,6 @@ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml -service zookeeper start -sleep 5 service clickhouse-server start && sleep 5 if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index b05e46406da..dbb7ae2477f 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -67,8 +67,6 @@ RUN apt-get --allow-unauthenticated update -y \ unixodbc-dev \ wget \ zlib1g-dev \ - zookeeper \ - zookeeperd RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index b6b48cd0943..adec0f51490 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -48,8 +48,6 @@ fi ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml -service zookeeper start -sleep 5 service clickhouse-server start && sleep 5 if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then diff --git a/docker/test/stateless_with_coverage/Dockerfile b/docker/test/stateless_with_coverage/Dockerfile index 77357d5142f..b76989de1cf 100644 --- a/docker/test/stateless_with_coverage/Dockerfile +++ b/docker/test/stateless_with_coverage/Dockerfile @@ -11,8 +11,6 @@ RUN apt-get update -y \ tzdata \ fakeroot \ debhelper \ - zookeeper \ - zookeeperd \ expect \ python \ python-lxml \ diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh index c3ccb18659b..6ab997925e4 100755 --- a/docker/test/stateless_with_coverage/run.sh +++ b/docker/test/stateless_with_coverage/run.sh @@ -72,9 +72,6 @@ ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-clien ln -s --backup=simple --suffix=_original.xml \ /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ -service zookeeper start -sleep 5 - start_clickhouse sleep 10 From bb8a2e161792b31188caf591b1ec7ce51cf267b2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 11:18:36 +0300 Subject: [PATCH 310/390] Better clickhouse-configs --- debian/rules | 4 +-- docker/test/fasttest/run.sh | 30 ++-------------- docker/test/stateful/run.sh | 22 ++---------- docker/test/stateful_with_coverage/run.sh | 24 ++----------- docker/test/stateless/run.sh | 41 ++-------------------- docker/test/stateless_unbundled/Dockerfile | 2 +- docker/test/stateless_unbundled/run.sh | 41 ++-------------------- docker/test/stateless_with_coverage/run.sh | 34 ++---------------- docker/test/stress/run.sh | 5 ++- 9 files changed, 17 insertions(+), 186 deletions(-) diff --git a/debian/rules b/debian/rules index 5b271a8691f..4387053c5aa 100755 --- a/debian/rules +++ b/debian/rules @@ -36,8 +36,8 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -DEB_CC ?= $(shell which gcc-9 gcc-8 gcc | head -n1) -DEB_CXX ?= $(shell which g++-9 g++-8 g++ | head -n1) +DEB_CC ?= $(shell which gcc-10 gcc-9 gcc-8 gcc | head -n1) +DEB_CXX ?= $(shell which g++-10 g++-9 g++-8 g++ | head -n1) ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ca691e5303c..165bee80b0c 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -111,35 +111,9 @@ ln -s /test_output /var/log/clickhouse-server cp "$CLICKHOUSE_DIR/programs/server/config.xml" /etc/clickhouse-server/ cp "$CLICKHOUSE_DIR/programs/server/users.xml" /etc/clickhouse-server/ -mkdir -p /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/ -#ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ -ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml +# install tests config +$CLICKHOUSE_DIR/tests/config/install.sh -# Keep original query_masking_rules.xml -ln -s --backup=simple --suffix=_original.xml /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ # Kill the server in case we are running locally and not in docker kill_clickhouse diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c3576acc0e4..87cc4054ee6 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -8,26 +8,8 @@ dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb dpkg -i package_folder/clickhouse-test_*.deb -mkdir -p /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ - -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ -fi +# install test configs +/usr/share/clickhouse-test/config/install.sh function start() { diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh index c2434b319b9..7191745ec83 100755 --- a/docker/test/stateful_with_coverage/run.sh +++ b/docker/test/stateful_with_coverage/run.sh @@ -48,28 +48,8 @@ mkdir -p /var/lib/clickhouse mkdir -p /var/log/clickhouse-server chmod 777 -R /var/log/clickhouse-server/ -# Temorary way to keep CI green while moving dictionaries to separate directory -mkdir -p /etc/clickhouse-server/dict_examples -chmod 777 -R /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ - ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ - ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; - -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ - -# Retain any pre-existing config and allow ClickHouse to load those if required -ln -s --backup=simple --suffix=_original.xml \ - /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ +# install test configs +/usr/share/clickhouse-test/config/install.sh function start() { diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index adec0f51490..9f2bb9bf62d 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -8,45 +8,8 @@ dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb dpkg -i package_folder/clickhouse-test_*.deb -mkdir -p /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ - -# Retain any pre-existing config and allow ClickHouse to load it if required -ln -s --backup=simple --suffix=_original.xml \ - /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ - -if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ -fi -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ -fi - -ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml +# install test configs +/usr/share/clickhouse-test/config/install.sh service clickhouse-server start && sleep 5 diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index dbb7ae2477f..cb8cd158e5f 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -66,7 +66,7 @@ RUN apt-get --allow-unauthenticated update -y \ unixodbc \ unixodbc-dev \ wget \ - zlib1g-dev \ + zlib1g-dev RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index adec0f51490..9f2bb9bf62d 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -8,45 +8,8 @@ dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb dpkg -i package_folder/clickhouse-test_*.deb -mkdir -p /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/ -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ - -# Retain any pre-existing config and allow ClickHouse to load it if required -ln -s --backup=simple --suffix=_original.xml \ - /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ - -if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/polymorphic_parts.xml /etc/clickhouse-server/config.d/ -fi -if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/ - ln -s /usr/share/clickhouse-test/config/database_atomic_usersd.xml /etc/clickhouse-server/users.d/ -fi - -ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml +# install test configs +/usr/share/clickhouse-test/config/install.sh service clickhouse-server start && sleep 5 diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh index 6ab997925e4..2f3f05a335a 100755 --- a/docker/test/stateless_with_coverage/run.sh +++ b/docker/test/stateless_with_coverage/run.sh @@ -39,38 +39,8 @@ mkdir -p /var/log/clickhouse-server chmod 777 -R /var/lib/clickhouse chmod 777 -R /var/log/clickhouse-server/ -# Temorary way to keep CI green while moving dictionaries to separate directory -mkdir -p /etc/clickhouse-server/dict_examples -chmod 777 -R /etc/clickhouse-server/dict_examples -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/; \ - ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/; \ - ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/; - -ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/ -ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/ -ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml - -# Retain any pre-existing config and allow ClickHouse to load it if required -ln -s --backup=simple --suffix=_original.xml \ - /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/ +# install test configs +/usr/share/clickhouse-test/config/install.sh start_clickhouse diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 8295e90b3ef..28c66a72d39 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -39,9 +39,8 @@ function start() done } -ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/ -ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/ -ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/ +# install test configs +/usr/share/clickhouse-test/config/install.sh echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment From 70e6b4093f895e6f8250d5492bc5b597876daf2b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 11:34:20 +0300 Subject: [PATCH 311/390] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 69391ca9fd4..6d1c6444d1b 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -145,3 +145,4 @@ 01461_query_start_time_microseconds 01455_shard_leaf_max_rows_bytes_to_read 01505_distributed_local_type_conversion_enum +01505_log_distributed_deadlock From 0936a3c1f893f6981f15a6f87d5daec4f61ca319 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 11:35:16 +0300 Subject: [PATCH 312/390] Install script for CI --- debian/rules | 4 ++-- release | 10 ++++---- tests/config/install.sh | 52 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100755 tests/config/install.sh diff --git a/debian/rules b/debian/rules index 4387053c5aa..837f81dd503 100755 --- a/debian/rules +++ b/debian/rules @@ -36,8 +36,8 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -DEB_CC ?= $(shell which gcc-10 gcc-9 gcc-8 gcc | head -n1) -DEB_CXX ?= $(shell which g++-10 g++-9 g++-8 g++ | head -n1) +DEB_CC ?= $(shell which gcc-10 gcc-9 gcc | head -n1) +DEB_CXX ?= $(shell which g++-10 g++-9 g++ | head -n1) ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) diff --git a/release b/release index b20683a9caa..b446ceca0d5 100755 --- a/release +++ b/release @@ -95,9 +95,9 @@ then exit 3 fi - export DEB_CC=${DEB_CC=clang-6.0} - export DEB_CXX=${DEB_CXX=clang++-6.0} - EXTRAPACKAGES="$EXTRAPACKAGES clang-6.0 lld-6.0" + export DEB_CC=${DEB_CC=clang-10} + export DEB_CXX=${DEB_CXX=clang++-10} + EXTRAPACKAGES="$EXTRAPACKAGES clang-10 lld-10" elif [[ $BUILD_TYPE == 'valgrind' ]]; then MALLOC_OPTS="-DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0" VERSION_POSTFIX+="+valgrind" @@ -118,8 +118,8 @@ echo -e "\nCurrent version is $VERSION_STRING" if [ -z "$NO_BUILD" ] ; then gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" if [ -z "$USE_PBUILDER" ] ; then - DEB_CC=${DEB_CC:=`which gcc-9 gcc-8 gcc | head -n1`} - DEB_CXX=${DEB_CXX:=`which g++-9 g++-8 g++ | head -n1`} + DEB_CC=${DEB_CC:=`which gcc-10 gcc-9 gcc | head -n1`} + DEB_CXX=${DEB_CXX:=`which gcc-10 g++-9 g++ | head -n1`} # Build (only binary packages). debuild --preserve-env -e PATH \ -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \ diff --git a/tests/config/install.sh b/tests/config/install.sh new file mode 100755 index 00000000000..7a005cab5ce --- /dev/null +++ b/tests/config/install.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# script allows to install configs for clickhouse server and clients required +# for testing (stateless and stateful tests) + +set -x -e + +DEST_SERVER_PATH="${1:-/etc/clickhouse-server}" +DEST_CLIENT_PATH="${2:-/etc/clickhouse-client}" +SRC_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +echo "Going to install test configs from $SRC_PATH into DEST_SERVER_PATH" + +mkdir -p $DEST_SERVER_PATH/config.d/ +mkdir -p $DEST_SERVER_PATH/users.d/ +mkdir -p $DEST_CLIENT_PATH + +ln -s $SRC_PATH/zookeeper.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/listen.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/part_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/text_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/metric_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/log_queries.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/readonly.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/access_management.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ +ln -s $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ +ln -s $SRC_PATH/decimals_dictionary.xml $DEST_SERVER_PATH/ +ln -s $SRC_PATH/executable_dictionary.xml $DEST_SERVER_PATH/ +ln -s $SRC_PATH/macros.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/disks.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/secure_ports.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/clusters.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/graphite.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/server.key $DEST_SERVER_PATH/ +ln -s $SRC_PATH/server.crt $DEST_SERVER_PATH/ +ln -s $SRC_PATH/dhparam.pem $DEST_SERVER_PATH/ + +# Retain any pre-existing config and allow ClickHouse to load it if required +ln -s --backup=simple --suffix=_original.xml \ + $SRC_PATH/query_masking_rules.xml $DEST_SERVER_PATH/config.d/ + +if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then + ln -s $SRC_PATH/polymorphic_parts.xml $DEST_SERVER_PATH/config.d/ +fi +if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then + ln -s $SRC_PATH/database_atomic_configd.xml $DEST_SERVER_PATH/config.d/ + ln -s $SRC_PATH/database_atomic_usersd.xml $DEST_SERVER_PATH/users.d/ +fi + +ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml From 8cbfd69adf413364e7c0264480ed692c73eb53d5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 11:35:35 +0300 Subject: [PATCH 313/390] Update decimal_casts.xml --- tests/performance/decimal_casts.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 7f609c8bd9d..582672fa30e 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,4 +1,4 @@ - + 15G From 4f3eb4532ace4f133b1a680e424e88e63438cb1a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 11:37:20 +0300 Subject: [PATCH 314/390] Add missed changes --- docker/test/fasttest/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 165bee80b0c..cf8dbc66870 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -113,6 +113,8 @@ cp "$CLICKHOUSE_DIR/programs/server/users.xml" /etc/clickhouse-server/ # install tests config $CLICKHOUSE_DIR/tests/config/install.sh +# doesn't support SSL +rm -f /etc/clickhouse-server/config.d/secure_ports.xml # Kill the server in case we are running locally and not in docker From a2845acf37b961c8d4b990cc900e1fba7ebec8be Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 24 Sep 2020 16:38:57 +0800 Subject: [PATCH 315/390] Fix joinGet crash with LowCardinality types. --- src/Functions/FunctionJoinGet.h | 2 +- .../0_stateless/01400_join_get_with_multi_keys.reference | 1 + .../queries/0_stateless/01400_join_get_with_multi_keys.sql | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 6b3b1202f60..e1afd2715f0 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -80,7 +80,7 @@ public: DataTypePtr getReturnType(const ColumnsWithTypeAndName &) const override { return {}; } // Not used bool useDefaultImplementationForNulls() const override { return false; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } diff --git a/tests/queries/0_stateless/01400_join_get_with_multi_keys.reference b/tests/queries/0_stateless/01400_join_get_with_multi_keys.reference index 49d59571fbf..726b0a9a7a5 100644 --- a/tests/queries/0_stateless/01400_join_get_with_multi_keys.reference +++ b/tests/queries/0_stateless/01400_join_get_with_multi_keys.reference @@ -1 +1,2 @@ 0.1 +0.1 diff --git a/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql b/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql index 73068270762..8a19865359b 100644 --- a/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql +++ b/tests/queries/0_stateless/01400_join_get_with_multi_keys.sql @@ -6,4 +6,10 @@ INSERT INTO test_joinGet VALUES ('ab', '1', 0.1), ('ab', '2', 0.2), ('cd', '3', SELECT joinGet(test_joinGet, 'c', 'ab', '1'); +CREATE TABLE test_lc(a LowCardinality(String), b LowCardinality(String), c Float64) ENGINE = Join(any, left, a, b); + +INSERT INTO test_lc VALUES ('ab', '1', 0.1), ('ab', '2', 0.2), ('cd', '3', 0.3); + +SELECT joinGet(test_lc, 'c', 'ab', '1'); + DROP TABLE test_joinGet; From 8adb9c922482573716c15ca4f063bc5eed7d9676 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Thu, 24 Sep 2020 12:39:55 +0300 Subject: [PATCH 316/390] test for fixed issue (#15189) --- .../01482_move_to_prewhere_and_cast.reference | 1 + .../01482_move_to_prewhere_and_cast.sql | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference create mode 100644 tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql diff --git a/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference new file mode 100644 index 00000000000..29597554bbc --- /dev/null +++ b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference @@ -0,0 +1 @@ +ApplicationA 2020-01-01 diff --git a/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql new file mode 100644 index 00000000000..b79a3cf05b4 --- /dev/null +++ b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS APPLICATION; +DROP TABLE IF EXISTS DATABASE_IO; + +CREATE TABLE APPLICATION ( + `Name` LowCardinality(String), + `Base` LowCardinality(String) +) ENGINE = Memory(); + +insert into table APPLICATION values ('ApplicationA', 'BaseA'), ('ApplicationB', 'BaseB') , ('ApplicationC', 'BaseC'); + +CREATE TABLE DATABASE_IO ( + `Application` LowCardinality(String), + `Base` LowCardinality(String), + `Date` DateTime, + `Ios` UInt32 ) +ENGINE = MergeTree() +ORDER BY Date; + +insert into table DATABASE_IO values ('AppA', 'BaseA', '2020-01-01 00:00:00', 1000); + +SELECT `APPLICATION`.`Name` AS `App`, + CAST(CAST(`DATABASE_IO`.`Date` AS DATE) AS DATE) AS `date` +FROM `DATABASE_IO` +INNER +JOIN `APPLICATION` ON (`DATABASE_IO`.`Base` = `APPLICATION`.`Base`) +WHERE ( + CAST(CAST(`DATABASE_IO`.`Date` AS DATE) AS TIMESTAMP) >= toDateTime('2020-01-01 00:00:00') +); + +DROP TABLE APPLICATION; +DROP TABLE DATABASE_IO; From b8861ad6cc00cd60330f253dac999053dde8ce92 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:21:34 +0300 Subject: [PATCH 317/390] Move config files to directories --- tests/config/{ => config.d}/clusters.xml | 0 tests/config/{ => config.d}/custom_settings_prefixes.xml | 0 tests/config/{ => config.d}/database_atomic_configd.xml | 0 tests/config/{ => config.d}/disks.xml | 0 tests/config/{ => config.d}/graphite.xml | 0 tests/config/{ => config.d}/listen.xml | 0 tests/config/{ => config.d}/macros.xml | 0 tests/config/{ => config.d}/metric_log.xml | 0 tests/config/{ => config.d}/part_log.xml | 0 tests/config/{ => config.d}/polymorphic_parts.xml | 0 tests/config/{ => config.d}/query_masking_rules.xml | 0 tests/config/{ => config.d}/secure_ports.xml | 0 tests/config/{ => config.d}/text_log.xml | 0 tests/config/{ => config.d}/zookeeper.xml | 0 tests/config/{ => users.d}/access_management.xml | 0 tests/config/{ => users.d}/database_atomic_usersd.xml | 0 tests/config/{ => users.d}/log_queries.xml | 0 tests/config/{ => users.d}/readonly.xml | 0 18 files changed, 0 insertions(+), 0 deletions(-) rename tests/config/{ => config.d}/clusters.xml (100%) rename tests/config/{ => config.d}/custom_settings_prefixes.xml (100%) rename tests/config/{ => config.d}/database_atomic_configd.xml (100%) rename tests/config/{ => config.d}/disks.xml (100%) rename tests/config/{ => config.d}/graphite.xml (100%) rename tests/config/{ => config.d}/listen.xml (100%) rename tests/config/{ => config.d}/macros.xml (100%) rename tests/config/{ => config.d}/metric_log.xml (100%) rename tests/config/{ => config.d}/part_log.xml (100%) rename tests/config/{ => config.d}/polymorphic_parts.xml (100%) rename tests/config/{ => config.d}/query_masking_rules.xml (100%) rename tests/config/{ => config.d}/secure_ports.xml (100%) rename tests/config/{ => config.d}/text_log.xml (100%) rename tests/config/{ => config.d}/zookeeper.xml (100%) rename tests/config/{ => users.d}/access_management.xml (100%) rename tests/config/{ => users.d}/database_atomic_usersd.xml (100%) rename tests/config/{ => users.d}/log_queries.xml (100%) rename tests/config/{ => users.d}/readonly.xml (100%) diff --git a/tests/config/clusters.xml b/tests/config/config.d/clusters.xml similarity index 100% rename from tests/config/clusters.xml rename to tests/config/config.d/clusters.xml diff --git a/tests/config/custom_settings_prefixes.xml b/tests/config/config.d/custom_settings_prefixes.xml similarity index 100% rename from tests/config/custom_settings_prefixes.xml rename to tests/config/config.d/custom_settings_prefixes.xml diff --git a/tests/config/database_atomic_configd.xml b/tests/config/config.d/database_atomic_configd.xml similarity index 100% rename from tests/config/database_atomic_configd.xml rename to tests/config/config.d/database_atomic_configd.xml diff --git a/tests/config/disks.xml b/tests/config/config.d/disks.xml similarity index 100% rename from tests/config/disks.xml rename to tests/config/config.d/disks.xml diff --git a/tests/config/graphite.xml b/tests/config/config.d/graphite.xml similarity index 100% rename from tests/config/graphite.xml rename to tests/config/config.d/graphite.xml diff --git a/tests/config/listen.xml b/tests/config/config.d/listen.xml similarity index 100% rename from tests/config/listen.xml rename to tests/config/config.d/listen.xml diff --git a/tests/config/macros.xml b/tests/config/config.d/macros.xml similarity index 100% rename from tests/config/macros.xml rename to tests/config/config.d/macros.xml diff --git a/tests/config/metric_log.xml b/tests/config/config.d/metric_log.xml similarity index 100% rename from tests/config/metric_log.xml rename to tests/config/config.d/metric_log.xml diff --git a/tests/config/part_log.xml b/tests/config/config.d/part_log.xml similarity index 100% rename from tests/config/part_log.xml rename to tests/config/config.d/part_log.xml diff --git a/tests/config/polymorphic_parts.xml b/tests/config/config.d/polymorphic_parts.xml similarity index 100% rename from tests/config/polymorphic_parts.xml rename to tests/config/config.d/polymorphic_parts.xml diff --git a/tests/config/query_masking_rules.xml b/tests/config/config.d/query_masking_rules.xml similarity index 100% rename from tests/config/query_masking_rules.xml rename to tests/config/config.d/query_masking_rules.xml diff --git a/tests/config/secure_ports.xml b/tests/config/config.d/secure_ports.xml similarity index 100% rename from tests/config/secure_ports.xml rename to tests/config/config.d/secure_ports.xml diff --git a/tests/config/text_log.xml b/tests/config/config.d/text_log.xml similarity index 100% rename from tests/config/text_log.xml rename to tests/config/config.d/text_log.xml diff --git a/tests/config/zookeeper.xml b/tests/config/config.d/zookeeper.xml similarity index 100% rename from tests/config/zookeeper.xml rename to tests/config/config.d/zookeeper.xml diff --git a/tests/config/access_management.xml b/tests/config/users.d/access_management.xml similarity index 100% rename from tests/config/access_management.xml rename to tests/config/users.d/access_management.xml diff --git a/tests/config/database_atomic_usersd.xml b/tests/config/users.d/database_atomic_usersd.xml similarity index 100% rename from tests/config/database_atomic_usersd.xml rename to tests/config/users.d/database_atomic_usersd.xml diff --git a/tests/config/log_queries.xml b/tests/config/users.d/log_queries.xml similarity index 100% rename from tests/config/log_queries.xml rename to tests/config/users.d/log_queries.xml diff --git a/tests/config/readonly.xml b/tests/config/users.d/readonly.xml similarity index 100% rename from tests/config/readonly.xml rename to tests/config/users.d/readonly.xml From 2846645fa3482aae0b6469f810ec531e32d42fdf Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:22:00 +0300 Subject: [PATCH 318/390] Update install.sh --- tests/config/install.sh | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/config/install.sh b/tests/config/install.sh index 7a005cab5ce..20641833815 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -9,44 +9,46 @@ DEST_SERVER_PATH="${1:-/etc/clickhouse-server}" DEST_CLIENT_PATH="${2:-/etc/clickhouse-client}" SRC_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -echo "Going to install test configs from $SRC_PATH into DEST_SERVER_PATH" +echo "Going to install test configs from $SRC_PATH into $DEST_SERVER_PATH" mkdir -p $DEST_SERVER_PATH/config.d/ mkdir -p $DEST_SERVER_PATH/users.d/ mkdir -p $DEST_CLIENT_PATH -ln -s $SRC_PATH/zookeeper.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/listen.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/part_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/text_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/metric_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/log_queries.xml $DEST_SERVER_PATH/users.d/ -ln -s $SRC_PATH/readonly.xml $DEST_SERVER_PATH/users.d/ -ln -s $SRC_PATH/access_management.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/config.d/zookeeper.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/listen.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/part_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/text_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/metric_log.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/macros.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/disks.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/secure_ports.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/clusters.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/config.d/graphite.xml $DEST_SERVER_PATH/config.d/ +ln -s $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ +ln -s $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ + ln -s $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ ln -s $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ ln -s $SRC_PATH/decimals_dictionary.xml $DEST_SERVER_PATH/ ln -s $SRC_PATH/executable_dictionary.xml $DEST_SERVER_PATH/ -ln -s $SRC_PATH/macros.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/disks.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/secure_ports.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/clusters.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/graphite.xml $DEST_SERVER_PATH/config.d/ + ln -s $SRC_PATH/server.key $DEST_SERVER_PATH/ ln -s $SRC_PATH/server.crt $DEST_SERVER_PATH/ ln -s $SRC_PATH/dhparam.pem $DEST_SERVER_PATH/ # Retain any pre-existing config and allow ClickHouse to load it if required ln -s --backup=simple --suffix=_original.xml \ - $SRC_PATH/query_masking_rules.xml $DEST_SERVER_PATH/config.d/ + $SRC_PATH/config.d/query_masking_rules.xml $DEST_SERVER_PATH/config.d/ if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then - ln -s $SRC_PATH/polymorphic_parts.xml $DEST_SERVER_PATH/config.d/ + ln -s $SRC_PATH/config.d/polymorphic_parts.xml $DEST_SERVER_PATH/config.d/ fi if [[ -n "$USE_DATABASE_ATOMIC" ]] && [[ "$USE_DATABASE_ATOMIC" -eq 1 ]]; then - ln -s $SRC_PATH/database_atomic_configd.xml $DEST_SERVER_PATH/config.d/ - ln -s $SRC_PATH/database_atomic_usersd.xml $DEST_SERVER_PATH/users.d/ + ln -s $SRC_PATH/config.d/database_atomic_configd.xml $DEST_SERVER_PATH/config.d/ + ln -s $SRC_PATH/users.d/database_atomic_usersd.xml $DEST_SERVER_PATH/users.d/ fi ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml From 24cb88f808471a65ce4c4a183a5284bcd0177177 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:28:13 +0300 Subject: [PATCH 319/390] Add readme --- tests/config/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/config/README.md diff --git a/tests/config/README.md b/tests/config/README.md new file mode 100644 index 00000000000..b172c8afea3 --- /dev/null +++ b/tests/config/README.md @@ -0,0 +1,8 @@ +# ClickHouse configs for test environment + +## How to use +If you want to run all tests from `tests/queries/0_stateless` and `test/queries/0_stateful` on your local machine you have to set up configs from this directory for your clickhouse-server. This most simple way is to install them using `install.sh` script. + +## How to add new config + +Just place file `.xml` with new config into appropriate directory and add `ln` command into `install.sh` script. After that CI will use this config in all tests runs. From 7277cbe1e69e60d8fbe1af225ecc10de2f5905b9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:28:56 +0300 Subject: [PATCH 320/390] Fix readme --- tests/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/README.md b/tests/config/README.md index b172c8afea3..7e082403913 100644 --- a/tests/config/README.md +++ b/tests/config/README.md @@ -1,7 +1,7 @@ # ClickHouse configs for test environment ## How to use -If you want to run all tests from `tests/queries/0_stateless` and `test/queries/0_stateful` on your local machine you have to set up configs from this directory for your clickhouse-server. This most simple way is to install them using `install.sh` script. +If you want to run all tests from `tests/queries/0_stateless` and `test/queries/1_stateful` on your local machine you have to set up configs from this directory for your clickhouse-server. This most simple way is to install them using `install.sh` script. ## How to add new config From 912d45c443220d238556059650498665e72d944f Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:31:21 +0300 Subject: [PATCH 321/390] Fix readme --- tests/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/README.md b/tests/config/README.md index 7e082403913..faa0e922f13 100644 --- a/tests/config/README.md +++ b/tests/config/README.md @@ -1,7 +1,7 @@ # ClickHouse configs for test environment ## How to use -If you want to run all tests from `tests/queries/0_stateless` and `test/queries/1_stateful` on your local machine you have to set up configs from this directory for your clickhouse-server. This most simple way is to install them using `install.sh` script. +CI use this configs in all checks installing them with `install.sh` script. If you want to run all tests from `tests/queries/0_stateless` and `test/queries/1_stateful` on your local machine you have to set up configs from this directory for your `clickhouse-server`. The most simple way is to install them using `install.sh` script. Other option is just copy files into your clickhouse config directory. ## How to add new config From f8ab9896e7142348d9c60087c77b4c16fcfd42fb Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 13:32:49 +0300 Subject: [PATCH 322/390] One more time --- tests/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/README.md b/tests/config/README.md index faa0e922f13..8dd775a275a 100644 --- a/tests/config/README.md +++ b/tests/config/README.md @@ -1,7 +1,7 @@ # ClickHouse configs for test environment ## How to use -CI use this configs in all checks installing them with `install.sh` script. If you want to run all tests from `tests/queries/0_stateless` and `test/queries/1_stateful` on your local machine you have to set up configs from this directory for your `clickhouse-server`. The most simple way is to install them using `install.sh` script. Other option is just copy files into your clickhouse config directory. +CI use these configs in all checks installing them with `install.sh` script. If you want to run all tests from `tests/queries/0_stateless` and `test/queries/1_stateful` on your local machine you have to set up configs from this directory for your `clickhouse-server`. The most simple way is to install them using `install.sh` script. Other option is just copy files into your clickhouse config directory. ## How to add new config From e65f1098e27a664e83ce319ae6938573e8df547b Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 24 Sep 2020 13:44:13 +0300 Subject: [PATCH 323/390] Bump CI. From e7a764c2c5a6d2334ff80a8362dea549d8117582 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 24 Sep 2020 13:49:36 +0300 Subject: [PATCH 324/390] fix flacky test --- .../0_stateless/01320_create_sync_race_condition_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index f72c9da27ba..cc6a66bd6bc 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -26,4 +26,4 @@ timeout 10 bash -c thread2 & wait -$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01320" 2>&1 | grep -v "New table appeared in database being dropped or detached. Try again." || exit 0 +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01320" 2>&1 | grep -F "Code:" | grep -v "New table appeared in database being dropped or detached" || exit 0 From 41ad365eafaee7cdba978967414467dc9b8b18f4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 14:11:59 +0300 Subject: [PATCH 325/390] Increase fast test threads to 8 Tests don't often fail in parallel (1/10 runs), so we can make them faster. --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ca691e5303c..4efacc7cb9e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -216,7 +216,7 @@ TESTS_TO_SKIP=( 01460_DistributedFilesToInsert ) -clickhouse-test -j 4 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt +clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt # substr is to remove semicolon after test name From 5f9ef1ebbf5acf0d3b616abfd1f68ba12704b0fd Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 24 Sep 2020 14:50:18 +0300 Subject: [PATCH 326/390] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1c8e17086b..6b909dd710c 100644 --- a/README.md +++ b/README.md @@ -17,5 +17,5 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [eBay migrating from Druid](https://us02web.zoom.us/webinar/register/tZMkfu6rpjItHtaQ1DXcgPWcSOnmM73HLGKL) on September 23, 2020. * [ClickHouse for Edge Analytics](https://ones2020.sched.com/event/bWPs) on September 29, 2020. +* [ClickHouse online meetup (in Russian)](https://clck.ru/R2zB9) on October 1, 2020. From cc305cf88667009943dbfc43c2bcd42b8cbc9755 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 24 Sep 2020 15:34:03 +0300 Subject: [PATCH 327/390] Fix fuzzer image --- docker/test/fuzzer/run-fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index bcac5a433cc..d35a13cc421 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -48,7 +48,7 @@ function configure cp -av "$repo_dir"/programs/server/config* db cp -av "$repo_dir"/programs/server/user* db # TODO figure out which ones are needed - cp -av "$repo_dir"/tests/config/listen.xml db/config.d + cp -av "$repo_dir"/tests/config/config.d/listen.xml db/config.d cp -av "$script_dir"/query-fuzzer-tweaks-users.xml db/users.d } From c298c633a793e52543f38df78ef0e8098be6f0d6 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 24 Sep 2020 16:10:24 +0300 Subject: [PATCH 328/390] Revert "Test and doc for PR12771 krb5 + cyrus-sasl + kerberized kafka" --- .gitmodules | 1 - cmake/find/rdkafka.cmake | 4 +- contrib/cyrus-sasl | 2 +- docker/images.json | 4 - docker/test/integration/base/Dockerfile | 3 +- .../test/integration/kerberos_kdc/Dockerfile | 15 -- .../docker_compose_kerberized_kafka.yml | 59 ------- .../integration/runner/dockerd-entrypoint.sh | 1 - .../table-engines/integrations/kafka.md | 16 -- tests/integration/helpers/cluster.py | 40 +---- tests/integration/runner | 2 - .../test_storage_kerberized_kafka/__init__.py | 0 .../clickhouse_path/EMPTY_DIR | 0 .../configs/kafka.xml | 26 ---- .../configs/log_conf.xml | 11 -- .../kerberos_image_config.sh | 132 ---------------- .../secrets/broker_jaas.conf | 14 -- .../secrets/krb.conf | 22 --- .../secrets/zookeeper_jaas.conf | 14 -- .../test_storage_kerberized_kafka/test.py | 146 ------------------ 20 files changed, 7 insertions(+), 505 deletions(-) delete mode 100644 docker/test/integration/kerberos_kdc/Dockerfile delete mode 100644 docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml delete mode 100644 tests/integration/test_storage_kerberized_kafka/__init__.py delete mode 100644 tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR delete mode 100644 tests/integration/test_storage_kerberized_kafka/configs/kafka.xml delete mode 100644 tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml delete mode 100644 tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh delete mode 100644 tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf delete mode 100644 tests/integration/test_storage_kerberized_kafka/secrets/krb.conf delete mode 100644 tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf delete mode 100644 tests/integration/test_storage_kerberized_kafka/test.py diff --git a/.gitmodules b/.gitmodules index 865a876b276..eb21c4bfd00 100644 --- a/.gitmodules +++ b/.gitmodules @@ -186,4 +186,3 @@ [submodule "contrib/cyrus-sasl"] path = contrib/cyrus-sasl url = https://github.com/cyrusimap/cyrus-sasl - branch = cyrus-sasl-2.1 diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index ac11322f408..d9f815dbcdd 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -14,10 +14,10 @@ if (NOT ENABLE_RDKAFKA) return() endif() -if (NOT ARCH_ARM) +if (NOT ARCH_ARM AND USE_LIBGSASL) option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) elseif(USE_INTERNAL_RDKAFKA_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM}") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM} AND USE_LIBGSASL=${USE_LIBGSASL}") endif () if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt") diff --git a/contrib/cyrus-sasl b/contrib/cyrus-sasl index 9995bf9d8e1..6054630889f 160000 --- a/contrib/cyrus-sasl +++ b/contrib/cyrus-sasl @@ -1 +1 @@ -Subproject commit 9995bf9d8e14f58934d9313ac64f13780d6dd3c9 +Subproject commit 6054630889fd1cd8d0659573d69badcee1e23a00 diff --git a/docker/images.json b/docker/images.json index e9e91864e1e..8c2cb35b004 100644 --- a/docker/images.json +++ b/docker/images.json @@ -133,10 +133,6 @@ "name": "yandex/clickhouse-postgresql-java-client", "dependent": [] }, - "docker/test/integration/kerberos_kdc": { - "name": "yandex/clickhouse-kerberos-kdc", - "dependent": [] - }, "docker/test/base": { "name": "yandex/clickhouse-test-base", "dependent": [ diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 3e4e88965e0..35decd907c0 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -16,8 +16,7 @@ RUN apt-get update \ odbc-postgresql \ sqlite3 \ curl \ - tar \ - krb5-user + tar RUN rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ diff --git a/docker/test/integration/kerberos_kdc/Dockerfile b/docker/test/integration/kerberos_kdc/Dockerfile deleted file mode 100644 index ea231b1191d..00000000000 --- a/docker/test/integration/kerberos_kdc/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# docker build -t yandex/clickhouse-kerberos-kdc . - -FROM centos:6.6 -# old OS to make is faster and smaller - -RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation - -EXPOSE 88 749 - -RUN touch /config.sh -# should be overwritten e.g. via docker_compose volumes -# volumes: /some_path/my_kerberos_config.sh:/config.sh:ro - - -ENTRYPOINT ["/bin/bash", "/config.sh"] diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml deleted file mode 100644 index 3ce0000b148..00000000000 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '2.3' - -services: - kafka_kerberized_zookeeper: - image: confluentinc/cp-zookeeper:5.2.0 - # restart: always - hostname: kafka_kerberized_zookeeper - environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_SERVERS: "kafka_kerberized_zookeeper:2888:3888" - KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dsun.security.krb5.debug=true" - volumes: - - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets - - /dev/urandom:/dev/random - depends_on: - - kafka_kerberos - security_opt: - - label:disable - - kerberized_kafka1: - image: confluentinc/cp-kafka:5.2.0 - # restart: always - hostname: kerberized_kafka1 - ports: - - "9092:9092" - - "9093:9093" - environment: - KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://:9093 - KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:9093 - # KAFKA_LISTENERS: INSIDE://kerberized_kafka1:9092,OUTSIDE://kerberized_kafka1:19092 - # KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:9092,OUTSIDE://kerberized_kafka1:19092 - KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: GSSAPI - KAFKA_SASL_ENABLED_MECHANISMS: GSSAPI - KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: OUTSIDE:SASL_PLAINTEXT,UNSECURED_OUTSIDE:PLAINTEXT,UNSECURED_INSIDE:PLAINTEXT, - KAFKA_INTER_BROKER_LISTENER_NAME: OUTSIDE - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: "kafka_kerberized_zookeeper:2181" - KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true" - volumes: - - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets - - /dev/urandom:/dev/random - depends_on: - - kafka_kerberized_zookeeper - - kafka_kerberos - security_opt: - - label:disable - - kafka_kerberos: - image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG} - hostname: kafka_kerberos - volumes: - - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab - - ${KERBERIZED_KAFKA_DIR}/../../kerberos_image_config.sh:/config.sh - - /dev/urandom:/dev/random - ports: [88, 749] diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index cbdb7317b1e..c38260279ed 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -27,7 +27,6 @@ export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest} export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest} export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest} export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest} -export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest} cd /ClickHouse/tests/integration exec "$@" diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index d0a4bc928a7..fe9aa2ca25e 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -165,22 +165,6 @@ Similar to GraphiteMergeTree, the Kafka engine supports extended configuration u For a list of possible configuration options, see the [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). Use the underscore (`_`) instead of a dot in the ClickHouse configuration. For example, `check.crcs=true` will be `true`. -### Kerberos support {#kafka-kerberos-support} - -To deal with Kerberos-aware Kafka, add `security_protocol` child element with `sasl_plaintext` value. It is enough if Kerberos ticket-granting ticket is obtained and cached by OS facilities. -ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` and `sasl.kerberos.kinit.cmd` child elements. - -Example: - -``` xml - - - SASL_PLAINTEXT - /home/kafkauser/kafkauser.keytab - kafkauser/kafkahost@EXAMPLE.COM - -``` - ## Virtual Columns {#virtual-columns} - `_topic` — Kafka topic. diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 9ae24e3dafd..6d0f038daed 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -45,6 +45,7 @@ def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): f.write("=".join([var, value]) + "\n") return full_path + def subprocess_check_call(args): # Uncomment for debugging # print('run:', ' ' . join(args)) @@ -124,7 +125,6 @@ class ClickHouseCluster: self.base_zookeeper_cmd = None self.base_mysql_cmd = [] self.base_kafka_cmd = [] - self.base_kerberized_kafka_cmd = [] self.base_rabbitmq_cmd = [] self.base_cassandra_cmd = [] self.pre_zookeeper_commands = [] @@ -133,7 +133,6 @@ class ClickHouseCluster: self.with_mysql = False self.with_postgres = False self.with_kafka = False - self.with_kerberized_kafka = False self.with_rabbitmq = False self.with_odbc_drivers = False self.with_hdfs = False @@ -170,7 +169,7 @@ class ClickHouseCluster: def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None, macros=None, - with_zookeeper=False, with_mysql=False, with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, + with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, @@ -208,7 +207,6 @@ class ClickHouseCluster: zookeeper_config_path=self.zookeeper_config_path, with_mysql=with_mysql, with_kafka=with_kafka, - with_kerberized_kafka=with_kerberized_kafka, with_rabbitmq=with_rabbitmq, with_mongo=with_mongo, with_redis=with_redis, @@ -292,13 +290,6 @@ class ClickHouseCluster: p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] cmds.append(self.base_kafka_cmd) - if with_kerberized_kafka and not self.with_kerberized_kafka: - self.with_kerberized_kafka = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')]) - self.base_kerberized_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')] - cmds.append(self.base_kerberized_kafka_cmd) - if with_rabbitmq and not self.with_rabbitmq: self.with_rabbitmq = True self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]) @@ -617,11 +608,6 @@ class ClickHouseCluster: self.kafka_docker_id = self.get_instance_docker_id('kafka1') self.wait_schema_registry_to_start(120) - if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: - env = os.environ.copy() - env['KERBERIZED_KAFKA_DIR'] = instance.path + '/' - subprocess.check_call(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=env) - self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') if self.with_rabbitmq and self.base_rabbitmq_cmd: subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) self.rabbitmq_docker_id = self.get_instance_docker_id('rabbitmq1') @@ -802,12 +788,9 @@ services: - {instance_config_dir}:/etc/clickhouse-server/ - {db_dir}:/var/lib/clickhouse/ - {logs_dir}:/var/log/clickhouse-server/ - - /etc/passwd:/etc/passwd:ro {binary_volume} {odbc_bridge_volume} {odbc_ini_path} - {keytab_path} - {krb5_conf} entrypoint: {entrypoint_cmd} tmpfs: {tmpfs} cap_add: @@ -837,7 +820,7 @@ class ClickHouseInstance: def __init__( self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, custom_dictionaries, - macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_kerberized_kafka, with_rabbitmq, with_mongo, + macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, @@ -856,7 +839,6 @@ class ClickHouseInstance: self.custom_user_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_user_configs] self.custom_dictionaries_paths = [p.abspath(p.join(base_path, c)) for c in custom_dictionaries] self.clickhouse_path_dir = p.abspath(p.join(base_path, clickhouse_path_dir)) if clickhouse_path_dir else None - self.kerberos_secrets_dir = p.abspath(p.join(base_path, 'secrets')) self.macros = macros if macros is not None else {} self.with_zookeeper = with_zookeeper self.zookeeper_config_path = zookeeper_config_path @@ -866,7 +848,6 @@ class ClickHouseInstance: self.with_mysql = with_mysql self.with_kafka = with_kafka - self.with_kerberized_kafka = with_kerberized_kafka self.with_rabbitmq = with_rabbitmq self.with_mongo = with_mongo self.with_redis = with_redis @@ -882,13 +863,6 @@ class ClickHouseInstance: else: self.odbc_ini_path = "" - if with_kerberized_kafka: - self.keytab_path = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets:/tmp/keytab" - self.krb5_conf = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets/krb.conf:/etc/krb5.conf:ro" - else: - self.keytab_path = "" - self.krb5_conf = "" - self.docker_client = None self.ip_address = None self.client = None @@ -1218,9 +1192,6 @@ class ClickHouseInstance: if self.with_zookeeper: shutil.copy(self.zookeeper_config_path, conf_d_dir) - if self.with_kerberized_kafka: - shutil.copytree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) - # Copy config.d configs print "Copy custom test config files {} to {}".format(self.custom_main_config_paths, self.config_d_dir) for path in self.custom_main_config_paths: @@ -1256,9 +1227,6 @@ class ClickHouseInstance: depends_on.append("kafka1") depends_on.append("schema-registry") - if self.with_kerberized_kafka: - depends_on.append("kerberized_kafka1") - if self.with_rabbitmq: depends_on.append("rabbitmq1") @@ -1322,8 +1290,6 @@ class ClickHouseInstance: user=os.getuid(), env_file=env_file, odbc_ini_path=odbc_ini_path, - keytab_path=self.keytab_path, - krb5_conf=self.krb5_conf, entrypoint_cmd=entrypoint_cmd, networks=networks, app_net=app_net, diff --git a/tests/integration/runner b/tests/integration/runner index f38ab0aa042..f097a42e52a 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -156,8 +156,6 @@ if __name__ == "__main__": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) elif image == "yandex/clickhouse-integration-test": env_tags += "-e {}={}".format("DOCKER_BASE_TAG", tag) - elif image == "yandex/clickhouse-kerberos-kdc": - env_tags += "-e {}={}".format("DOCKER_KERBEROS_KDC_TAG", tag) else: logging.info("Unknown image {}".format(image)) diff --git a/tests/integration/test_storage_kerberized_kafka/__init__.py b/tests/integration/test_storage_kerberized_kafka/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR b/tests/integration/test_storage_kerberized_kafka/clickhouse_path/EMPTY_DIR deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml b/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml deleted file mode 100644 index 0302bd78e3f..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/configs/kafka.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - earliest - - SASL_PLAINTEXT - GSSAPI - kafka - /tmp/keytab/clickhouse.keytab - kafkauser/instance@TEST.CLICKHOUSE.TECH - security - false - - - - - 300 - - 6000 - - diff --git a/tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml b/tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml deleted file mode 100644 index 95466269afe..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/configs/log_conf.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - trace - /var/log/clickhouse-server/log.log - /var/log/clickhouse-server/log.err.log - 1000M - 10 - /var/log/clickhouse-server/stderr.log - /var/log/clickhouse-server/stdout.log - - \ No newline at end of file diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh deleted file mode 100644 index dda10d47d94..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash - - -set -x # trace - -: "${REALM:=TEST.CLICKHOUSE.TECH}" -: "${DOMAIN_REALM:=test.clickhouse.tech}" -: "${KERB_MASTER_KEY:=masterkey}" -: "${KERB_ADMIN_USER:=admin}" -: "${KERB_ADMIN_PASS:=admin}" - -create_config() { - : "${KDC_ADDRESS:=$(hostname -f)}" - - cat>/etc/krb5.conf</var/kerberos/krb5kdc/kdc.conf< /var/kerberos/krb5kdc/kadm5.acl -} - -create_keytabs() { - - kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@${REALM}" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@${REALM}" - - kadmin.local -q "addprinc -randkey kafka/kerberized_kafka1@${REALM}" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kerberized_kafka.keytab kafka/kerberized_kafka1@${REALM}" - - kadmin.local -q "addprinc -randkey zkclient@${REALM}" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/zkclient.keytab zkclient@${REALM}" - - - kadmin.local -q "addprinc -randkey kafkauser/instance@${REALM}" - kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab kafkauser/instance@${REALM}" - - chmod g+r /tmp/keytab/clickhouse.keytab - -} - -main() { - - if [ ! -f /kerberos_initialized ]; then - create_config - create_db - create_admin_user - start_kdc - - touch /kerberos_initialized - fi - - if [ ! -f /var/kerberos/krb5kdc/principal ]; then - while true; do sleep 1000; done - else - start_kdc - create_keytabs - tail -F /var/log/kerberos/krb5kdc.log - fi - -} - -[[ "$0" == "${BASH_SOURCE[0]}" ]] && main "$@" diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf b/tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf deleted file mode 100644 index 8a55ec2faa0..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/secrets/broker_jaas.conf +++ /dev/null @@ -1,14 +0,0 @@ -KafkaServer { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - keyTab="/etc/kafka/secrets/kerberized_kafka.keytab" - principal="kafka/kerberized_kafka1@TEST.CLICKHOUSE.TECH"; -}; -Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - keyTab="/etc/kafka/secrets/zkclient.keytab" - principal="zkclient@TEST.CLICKHOUSE.TECH"; -}; diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf deleted file mode 100644 index 1efdf510f22..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf +++ /dev/null @@ -1,22 +0,0 @@ -[logging] - default = FILE:/var/log/kerberos/krb5libs.log - kdc = FILE:/var/log/kerberos/krb5kdc.log - admin_server = FILE:/var/log/kerberos/kadmind.log - -[libdefaults] - default_realm = TEST.CLICKHOUSE.TECH - dns_lookup_realm = false - dns_lookup_kdc = false - ticket_lifetime = 15s - renew_lifetime = 15s - forwardable = true - -[realms] - TEST.CLICKHOUSE.TECH = { - kdc = kafka_kerberos - admin_server = kafka_kerberos - } - -[domain_realm] - .TEST.CLICKHOUSE.TECH = TEST.CLICKHOUSE.TECH - TEST.CLICKHOUSE.TECH = TEST.CLICKHOUSE.TECH diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf b/tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf deleted file mode 100644 index 1b1f8103f42..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/secrets/zookeeper_jaas.conf +++ /dev/null @@ -1,14 +0,0 @@ -Server { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - keyTab="/etc/kafka/secrets/kafka_kerberized_zookeeper.keytab" - principal="zookeeper/kafka_kerberized_zookeeper@TEST.CLICKHOUSE.TECH"; -}; -Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - keyTab="/etc/kafka/secrets/zkclient.keytab" - principal="zkclient@TEST.CLICKHOUSE.TECH"; -}; diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py deleted file mode 100644 index ec23d340977..00000000000 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ /dev/null @@ -1,146 +0,0 @@ -import os.path as p -import random -import threading -import time -import pytest - -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV -from helpers.client import QueryRuntimeException -from helpers.network import PartitionManager - -import json -import subprocess -import kafka.errors -from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection -from kafka.admin import NewTopic -from kafka.protocol.admin import DescribeGroupsResponse_v1, DescribeGroupsRequest_v1 -from kafka.protocol.group import MemberAssignment -import socket - -cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/kafka.xml', 'configs/log_conf.xml' ], - with_kerberized_kafka=True, - clickhouse_path_dir="clickhouse_path" - ) -kafka_id = '' # instance.cluster.kafka_docker_id - -# Helpers - -def check_kafka_is_available(): - - # plaintext - p = subprocess.Popen(('docker', - 'exec', - '-i', - kafka_id, - '/usr/bin/kafka-broker-api-versions', - '--bootstrap-server', - 'localhost:9093'), - stdout=subprocess.PIPE) - p.communicate() - return p.returncode == 0 - - -def wait_kafka_is_available(max_retries=50): - retries = 0 - while True: - if check_kafka_is_available(): - break - else: - retries += 1 - if retries > max_retries: - raise "Kafka is not available" - print("Waiting for Kafka to start up") - time.sleep(1) - - -def kafka_produce(topic, messages, timestamp=None): - producer = KafkaProducer(bootstrap_servers="localhost:9093") - for message in messages: - producer.send(topic=topic, value=message, timestamp_ms=timestamp) - producer.flush() - print ("Produced {} messages for topic {}".format(len(messages), topic)) - - - -# Fixtures - -@pytest.fixture(scope="module") -def kafka_cluster(): - try: - global kafka_id - cluster.start() - kafka_id = instance.cluster.kerberized_kafka_docker_id - print("kafka_id is {}".format(kafka_id)) - yield cluster - - finally: - cluster.shutdown() - - -@pytest.fixture(autouse=True) -def kafka_setup_teardown(): - instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') - wait_kafka_is_available() - print("kafka is available - running test") - yield # run test - -# Tests - -@pytest.mark.timeout(180) # wait to build containers -def test_kafka_json_as_string(kafka_cluster): - kafka_produce('kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) - - instance.query(''' - CREATE TABLE test.kafka (field String) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', - kafka_topic_list = 'kafka_json_as_string', - kafka_group_name = 'kafka_json_as_string', - kafka_format = 'JSONAsString', - kafka_flush_interval_ms=1000; - ''') - - result = instance.query('SELECT * FROM test.kafka;') - expected = '''\ -{"t": 123, "e": {"x": "woof"} } -{"t": 124, "e": {"x": "test"} } -{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} -''' - assert TSV(result) == TSV(expected) - assert instance.contains_in_log("Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") - -def test_kafka_json_as_string_no_kdc(kafka_cluster): - kafka_produce('kafka_json_as_string_no_kdc', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) - - kafka_cluster.pause_container('kafka_kerberos') - time.sleep(45) # wait for ticket expiration - - instance.query(''' - CREATE TABLE test.kafka_no_kdc (field String) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', - kafka_topic_list = 'kafka_json_as_string_no_kdc', - kafka_group_name = 'kafka_json_as_string_no_kdc', - kafka_format = 'JSONAsString', - kafka_flush_interval_ms=1000; - ''') - - result = instance.query('SELECT * FROM test.kafka_no_kdc;') - expected = '' - - kafka_cluster.unpause_container('kafka_kerberos') - - - assert TSV(result) == TSV(expected) - assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") - assert instance.contains_in_log("Ticket expired") - assert instance.contains_in_log("Kerberos ticket refresh failed") - - -if __name__ == '__main__': - cluster.start() - raw_input("Cluster created, press any key to destroy...") - cluster.shutdown() From 9000ec0331fe062552590d5b253540e5af039b4f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 24 Sep 2020 16:21:08 +0300 Subject: [PATCH 329/390] trigger CI again to run intergation tests From 651518566a527f8a0c225f1d8a130712c8b810ee Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 24 Sep 2020 16:24:44 +0300 Subject: [PATCH 330/390] performance comparison --- docker/test/performance-comparison/perf.py | 23 ++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 2e047e6fb84..23686091e45 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -24,10 +24,11 @@ def tsv_escape(s): parser = argparse.ArgumentParser(description='Run performance test.') # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.") -parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.") +parser.add_argument('--host', nargs='*', default=['localhost'], help="Space-separated list of server hostname(s). Corresponds to '--port' options.") +parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated list of server port(s). Corresponds to '--host' options.") parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.') parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') +parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') parser.add_argument('--print-settings', action='store_true', help='Print test settings and exit.') @@ -188,10 +189,20 @@ for t in threads: for t in threads: t.join() -# Run the queries in randomized order, but preserve their indexes as specified -# in the test XML. To avoid using too much time, limit the number of queries -# we run per test. -queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries or len(test_queries))) +queries_to_run = range(0, len(test_queries)) + +if args.max_queries: + # If specified, test a limited number of queries chosen at random. + queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries)) + +if args.queries_to_run: + # Run the specified queries, with some sanity check. + for i in args.queries_to_run: + if i < 0 or i >= len(test_queries): + print(f'There is no query no. "{i}" in this test, only [{0}-{len(test_queries) - 1}] are present') + exit(1) + + queries_to_run = args.queries_to_run # Run test queries. for query_index in queries_to_run: From dce6a436f37713224c242092446e1a9fcf8b4a4a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Sep 2020 19:35:17 +0300 Subject: [PATCH 331/390] Fix strange code --- src/Common/randomSeed.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Common/randomSeed.cpp b/src/Common/randomSeed.cpp index 8ad624febdd..ded224e56c3 100644 --- a/src/Common/randomSeed.cpp +++ b/src/Common/randomSeed.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -19,7 +20,7 @@ namespace DB DB::UInt64 randomSeed() { struct timespec times; - if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, ×)) + if (clock_gettime(CLOCK_MONOTONIC, ×)) DB::throwFromErrno("Cannot clock_gettime.", DB::ErrorCodes::CANNOT_CLOCK_GETTIME); /// Not cryptographically secure as time, pid and stack address can be predictable. @@ -27,7 +28,7 @@ DB::UInt64 randomSeed() SipHash hash; hash.update(times.tv_nsec); hash.update(times.tv_sec); - hash.update(getpid()); + hash.update(getThreadId()); hash.update(×); return hash.get64(); } From 4ed4205647fe08613275a70e14aafc8e9180d8be Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 19:54:09 +0300 Subject: [PATCH 332/390] Update run.sh --- docker/test/fasttest/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 7f3a102276e..39992c072e3 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -97,7 +97,7 @@ ccache --zero-stats ||: mkdir build cd build cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/cmake_log.txt -ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt +time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt ninja install | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/install_log.txt @@ -192,7 +192,7 @@ TESTS_TO_SKIP=( 01460_DistributedFilesToInsert ) -clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt +time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt # substr is to remove semicolon after test name @@ -210,7 +210,7 @@ then kill_clickhouse # Clean the data so that there is no interference from the previous test run. - rm -rvf /var/lib/clickhouse ||: + rm -rf /var/lib/clickhouse ||: mkdir /var/lib/clickhouse clickhouse-server --config /etc/clickhouse-server/config.xml --daemon From a4a5b96d4a0915ca74c41b9c8ed7952c46a1e150 Mon Sep 17 00:00:00 2001 From: alekseik1 <1alekseik1@gmail.com> Date: Thu, 24 Sep 2020 20:08:16 +0300 Subject: [PATCH 333/390] do not expose ClickHouse server until initialization is complete --- docker/server/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 8fc9c670b06..ba352c2bbc2 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -89,7 +89,8 @@ EOT fi if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then - $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG & + # Listen only on localhost until the initialization is done + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & pid="$!" # check if clickhouse is ready to accept connections From c6ee67d3d8cac7d6524ed2c1eef89446ffcfb72b Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 24 Sep 2020 20:27:37 +0300 Subject: [PATCH 334/390] Bump CI. [2] From 0306c586a2896097938571da8adb232a444cfadc Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 24 Sep 2020 21:40:50 +0300 Subject: [PATCH 335/390] Update skip_list.json --- tests/queries/skip_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index e4713b2d960..4c00602c5d8 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -97,6 +97,7 @@ "00604_show_create_database", /// UUID must be specified in ATTACH TABLE "01190_full_attach_syntax", + "01249_bad_arguments_for_bloom_filter", /// Assumes blocking DROP "01320_create_sync_race_condition", /// Internal distionary name is different From c268464f9f83d80a24e62d5bbd241e73c55dd8f6 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 24 Sep 2020 22:38:58 +0300 Subject: [PATCH 336/390] add some disabled tests from arcadia to skip_list.json --- tests/queries/skip_list.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 4c00602c5d8..834590a3453 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -103,5 +103,9 @@ /// Internal distionary name is different "01225_show_create_table_from_dictionary", "01224_no_superfluous_dict_reload" + ], + "polymorphic-parts": [ + "00933_test_fix_extra_seek_on_compressed_cache", + "00446_clear_column_in_partition_zookeeper" ] } From e51d9dd2e76a9c5eade12a6518ed294705598ea3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 02:24:58 +0300 Subject: [PATCH 337/390] Fix MSan report in QueryLog --- src/Interpreters/ProcessList.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index d86b5678f6d..018ddbcfa1d 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -401,7 +401,7 @@ void ProcessList::killAllQueries() QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_events, bool get_settings) const { - QueryStatusInfo res; + QueryStatusInfo res{}; res.query = query; res.client_info = client_info; From 13529265c45ee0b2e526ecd4d6f0e0fc797a2fb5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 25 Sep 2020 02:28:57 +0300 Subject: [PATCH 338/390] Revert "Avoid deadlocks in Log/TinyLog" --- src/Common/FileChecker.cpp | 5 -- src/Common/FileChecker.h | 4 +- src/Storages/StorageFile.cpp | 40 +++------ src/Storages/StorageFile.h | 2 +- src/Storages/StorageLog.cpp | 60 ++++--------- src/Storages/StorageLog.h | 4 +- src/Storages/StorageStripeLog.cpp | 45 +++------- src/Storages/StorageStripeLog.h | 2 +- src/Storages/StorageTinyLog.cpp | 87 ++++++------------- src/Storages/StorageTinyLog.h | 4 +- .../0_stateless/01499_log_deadlock.reference | 3 - .../0_stateless/01499_log_deadlock.sql | 26 ------ ...2_long_log_tinylog_deadlock_race.reference | 6 -- .../01502_long_log_tinylog_deadlock_race.sh | 85 ------------------ .../01505_log_distributed_deadlock.reference | 0 .../01505_log_distributed_deadlock.sql | 12 --- .../queries/0_stateless/arcadia_skip_list.txt | 1 - 17 files changed, 79 insertions(+), 307 deletions(-) delete mode 100644 tests/queries/0_stateless/01499_log_deadlock.reference delete mode 100644 tests/queries/0_stateless/01499_log_deadlock.sql delete mode 100644 tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference delete mode 100755 tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh delete mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.reference delete mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.sql diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index b306c3af990..6cbec3bda77 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -41,11 +41,6 @@ void FileChecker::setEmpty(const String & full_file_path) map[fileName(full_file_path)] = 0; } -FileChecker::Map FileChecker::getFileSizes() const -{ - return map; -} - CheckResults FileChecker::check() const { // Read the files again every time you call `check` - so as not to violate the constancy. diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h index 59e7331952e..015d4cadb07 100644 --- a/src/Common/FileChecker.h +++ b/src/Common/FileChecker.h @@ -27,12 +27,10 @@ public: /// The purpose of this function is to rollback a group of unfinished writes. void repair(); +private: /// File name -> size. using Map = std::map; - Map getFileSizes() const; - -private: void initialize(); void updateImpl(const String & file_path); void load(Map & local_map, const String & path) const; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 7b094f9bc06..cc47047dc78 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -52,7 +52,6 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int INCORRECT_FILE_NAME; extern const int FILE_DOESNT_EXIST; - extern const int TIMEOUT_EXCEEDED; } namespace @@ -200,17 +199,6 @@ StorageFile::StorageFile(CommonArguments args) setInMemoryMetadata(storage_metadata); } - -static std::chrono::seconds getLockTimeout(const Context & context) -{ - const Settings & settings = context.getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - class StorageFileSource : public SourceWithProgress { public: @@ -257,9 +245,7 @@ public: { if (storage->use_table_fd) { - unique_lock = std::unique_lock(storage->rwlock, getLockTimeout(context)); - if (!unique_lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + unique_lock = std::unique_lock(storage->rwlock); /// We could use common ReadBuffer and WriteBuffer in storage to leverage cache /// and add ability to seek unseekable files, but cache sync isn't supported. @@ -278,9 +264,7 @@ public: } else { - shared_lock = std::shared_lock(storage->rwlock, getLockTimeout(context)); - if (!shared_lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + shared_lock = std::shared_lock(storage->rwlock); } } @@ -389,8 +373,8 @@ private: bool finished_generate = false; - std::shared_lock shared_lock; - std::unique_lock unique_lock; + std::shared_lock shared_lock; + std::unique_lock unique_lock; }; @@ -433,7 +417,7 @@ Pipe StorageFile::read( for (size_t i = 0; i < num_streams; ++i) pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); return Pipe::unitePipes(std::move(pipes)); } @@ -445,16 +429,12 @@ public: explicit StorageFileBlockOutputStream( StorageFile & storage_, const StorageMetadataPtr & metadata_snapshot_, - std::unique_lock && lock_, const CompressionMethod compression_method, const Context & context) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(std::move(lock_)) + , lock(storage.rwlock) { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - std::unique_ptr naked_buffer = nullptr; if (storage.use_table_fd) { @@ -508,7 +488,7 @@ public: private: StorageFile & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; std::unique_ptr write_buf; BlockOutputStreamPtr writer; bool prefix_written{false}; @@ -526,7 +506,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) path = paths[0]; - return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}, + return std::make_shared(*this, metadata_snapshot, chooseCompressionMethod(path, compression_method), context); } @@ -549,6 +529,8 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; + std::unique_lock lock(rwlock); + Poco::File(Poco::Path(path_new).parent()).createDirectories(); Poco::File(paths[0]).renameTo(path_new); @@ -565,6 +547,8 @@ void StorageFile::truncate( if (paths.size() != 1) throw Exception("Can't truncate table '" + getStorageID().getNameForLogs() + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); + std::unique_lock lock(rwlock); + if (use_table_fd) { if (0 != ::ftruncate(table_fd, 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index babc56e3a11..ea70dcd5311 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -89,7 +89,7 @@ private: std::atomic table_fd_was_used{false}; /// To detect repeating reads from stdin off_t table_fd_init_offset = -1; /// Initial position of fd, used for repeating reads - mutable std::shared_timed_mutex rwlock; + mutable std::shared_mutex rwlock; Poco::Logger * log = &Poco::Logger::get("StorageFile"); }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 2fbce21655c..e437bfb05f1 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -39,7 +39,6 @@ namespace DB namespace ErrorCodes { - extern const int TIMEOUT_EXCEEDED; extern const int LOGICAL_ERROR; extern const int DUPLICATE_COLUMN; extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT; @@ -51,6 +50,7 @@ namespace ErrorCodes class LogSource final : public SourceWithProgress { public: + static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -116,16 +116,13 @@ private: class LogBlockOutputStream final : public IBlockOutputStream { public: - explicit LogBlockOutputStream( - StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) + explicit LogBlockOutputStream(StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(std::move(lock_)) + , lock(storage.rwlock) , marks_stream( storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite)) { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~LogBlockOutputStream() override @@ -152,7 +149,7 @@ public: private: StorageLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -510,11 +507,9 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) } -void StorageLog::loadMarks(std::chrono::seconds lock_timeout) +void StorageLog::loadMarks() { - std::unique_lock lock(rwlock, lock_timeout); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + std::unique_lock lock(rwlock); if (loaded_marks) return; @@ -557,6 +552,8 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & { assert(table_path != new_path_to_table_data); { + std::unique_lock lock(rwlock); + disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -572,6 +569,8 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { + std::shared_lock lock(rwlock); + files.clear(); file_count = 0; loaded_marks = false; @@ -611,17 +610,6 @@ const StorageLog::Marks & StorageLog::getMarksWithRealRowCount(const StorageMeta return it->second.marks; } - -static std::chrono::seconds getLockTimeout(const Context & context) -{ - const Settings & settings = context.getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - Pipe StorageLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -632,15 +620,11 @@ Pipe StorageLog::read( unsigned num_streams) { metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); - - auto lock_timeout = getLockTimeout(context); - loadMarks(lock_timeout); + loadMarks(); NamesAndTypesList all_columns = Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)); - std::shared_lock lock(rwlock, lock_timeout); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + std::shared_lock lock(rwlock); Pipes pipes; @@ -669,28 +653,18 @@ Pipe StorageLog::read( max_read_buffer_size)); } - /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe::unitePipes(std::move(pipes)); } -BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) +BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) { - auto lock_timeout = getLockTimeout(context); - loadMarks(lock_timeout); - - std::unique_lock lock(rwlock, lock_timeout); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - - return std::make_shared(*this, metadata_snapshot, std::move(lock)); + loadMarks(); + return std::make_shared(*this, metadata_snapshot); } -CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & context) +CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { - std::shared_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - + std::shared_lock lock(rwlock); return file_checker.check(); } diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 3553426b9e6..49fc9a576c5 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -83,7 +83,7 @@ private: DiskPtr disk; String table_path; - mutable std::shared_timed_mutex rwlock; + mutable std::shared_mutex rwlock; Files files; @@ -104,7 +104,7 @@ private: /// Read marks files if they are not already read. /// It is done lazily, so that with a large number of tables, the server starts quickly. /// You can not call with a write locked `rwlock`. - void loadMarks(std::chrono::seconds lock_timeout); + void loadMarks(); /** For normal columns, the number of rows in the block is specified in the marks. * For array columns and nested structures, there are more than one group of marks that correspond to different files diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 8ff8035c128..c4344cf6f1f 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -47,13 +47,13 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; - extern const int TIMEOUT_EXCEEDED; } class StripeLogSource final : public SourceWithProgress { public: + static Block getHeader( StorageStripeLog & storage, const StorageMetadataPtr & metadata_snapshot, @@ -157,11 +157,10 @@ private: class StripeLogBlockOutputStream final : public IBlockOutputStream { public: - explicit StripeLogBlockOutputStream( - StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) + explicit StripeLogBlockOutputStream(StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(std::move(lock_)) + , lock(storage.rwlock) , data_out_file(storage.table_path + "data.bin") , data_out_compressed(storage.disk->writeFile(data_out_file, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append)) , data_out(std::make_unique( @@ -171,8 +170,6 @@ public: , index_out(std::make_unique(*index_out_compressed)) , block_out(*data_out, 0, metadata_snapshot->getSampleBlock(), false, index_out.get(), storage.disk->getFileSize(data_out_file)) { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~StripeLogBlockOutputStream() override @@ -226,7 +223,7 @@ public: private: StorageStripeLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; String data_out_file; std::unique_ptr data_out_compressed; @@ -289,6 +286,8 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora { assert(table_path != new_path_to_table_data); { + std::unique_lock lock(rwlock); + disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -298,16 +297,6 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora } -static std::chrono::seconds getLockTimeout(const Context & context) -{ - const Settings & settings = context.getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - Pipe StorageStripeLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -317,9 +306,7 @@ Pipe StorageStripeLog::read( const size_t /*max_block_size*/, unsigned num_streams) { - std::shared_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + std::shared_lock lock(rwlock); metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); @@ -358,28 +345,24 @@ Pipe StorageStripeLog::read( } -BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) +BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) { - std::unique_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - - return std::make_shared(*this, metadata_snapshot, std::move(lock)); + return std::make_shared(*this, metadata_snapshot); } -CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & context) +CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { - std::shared_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - + std::shared_lock lock(rwlock); return file_checker.check(); } void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, const Context &, TableExclusiveLockHolder &) { + std::shared_lock lock(rwlock); + disk->clearDirectory(table_path); + file_checker = FileChecker{disk, table_path + "sizes.json"}; } diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h index ca3bfe4ff75..f88120a932e 100644 --- a/src/Storages/StorageStripeLog.h +++ b/src/Storages/StorageStripeLog.h @@ -67,7 +67,7 @@ private: size_t max_compress_block_size; FileChecker file_checker; - mutable std::shared_timed_mutex rwlock; + mutable std::shared_mutex rwlock; Poco::Logger * log; }; diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 4d646c7451e..0bdcab8abf4 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -13,7 +13,6 @@ #include #include -#include #include #include #include @@ -47,7 +46,6 @@ namespace DB namespace ErrorCodes { - extern const int TIMEOUT_EXCEEDED; extern const int DUPLICATE_COLUMN; extern const int INCORRECT_FILE_NAME; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -57,6 +55,7 @@ namespace ErrorCodes class TinyLogSource final : public SourceWithProgress { public: + static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -67,17 +66,10 @@ public: return Nested::flatten(res); } - TinyLogSource( - size_t block_size_, - const NamesAndTypesList & columns_, - StorageTinyLog & storage_, - size_t max_read_buffer_size_, - FileChecker::Map file_sizes_) + TinyLogSource(size_t block_size_, const NamesAndTypesList & columns_, StorageTinyLog & storage_, size_t max_read_buffer_size_) : SourceWithProgress(getHeader(columns_)) - , block_size(block_size_), columns(columns_), storage(storage_) - , max_read_buffer_size(max_read_buffer_size_), file_sizes(std::move(file_sizes_)) - { - } + , block_size(block_size_), columns(columns_), storage(storage_), lock(storage_.rwlock) + , max_read_buffer_size(max_read_buffer_size_) {} String getName() const override { return "TinyLog"; } @@ -88,21 +80,19 @@ private: size_t block_size; NamesAndTypesList columns; StorageTinyLog & storage; + std::shared_lock lock; bool is_finished = false; size_t max_read_buffer_size; - FileChecker::Map file_sizes; struct Stream { - Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_, size_t file_size) + Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_) : plain(disk->readFile(data_path, std::min(max_read_buffer_size_, disk->getFileSize(data_path)))), - limited(std::make_unique(*plain, file_size, false)), compressed(*plain) { } std::unique_ptr plain; - std::unique_ptr limited; CompressedReadBuffer compressed; }; @@ -120,14 +110,9 @@ private: class TinyLogBlockOutputStream final : public IBlockOutputStream { public: - explicit TinyLogBlockOutputStream( - StorageTinyLog & storage_, - const StorageMetadataPtr & metadata_snapshot_, - std::unique_lock && lock_) - : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(std::move(lock_)) + explicit TinyLogBlockOutputStream(StorageTinyLog & storage_, const StorageMetadataPtr & metadata_snapshot_) + : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(storage_.rwlock) { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~TinyLogBlockOutputStream() override @@ -155,7 +140,7 @@ public: private: StorageTinyLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -246,17 +231,13 @@ void TinyLogSource::readData(const String & name, const IDataType & type, IColum String stream_name = IDataType::getFileNameForStream(name, path); if (!streams.count(stream_name)) - { - String file_path = storage.files[stream_name].data_file_path; - streams[stream_name] = std::make_unique( - storage.disk, file_path, max_read_buffer_size, file_sizes[fileName(file_path)]); - } + streams[stream_name] = std::make_unique(storage.disk, storage.files[stream_name].data_file_path, max_read_buffer_size); return &streams[stream_name]->compressed; }; if (deserialize_states.count(name) == 0) - type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); + type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); type.deserializeBinaryBulkWithMultipleStreams(column, limit, settings, deserialize_states[name]); } @@ -429,6 +410,8 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage { assert(table_path != new_path_to_table_data); { + std::unique_lock lock(rwlock); + disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -441,16 +424,6 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage } -static std::chrono::seconds getLockTimeout(const Context & context) -{ - const Settings & settings = context.getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - Pipe StorageTinyLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -464,40 +437,28 @@ Pipe StorageTinyLog::read( // When reading, we lock the entire storage, because we only have one file // per column and can't modify it concurrently. - const Settings & settings = context.getSettingsRef(); - - std::shared_lock lock{rwlock, getLockTimeout(context)}; - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - - /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe(std::make_shared( - max_block_size, - Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), - *this, - settings.max_read_buffer_size, - file_checker.getFileSizes())); + max_block_size, Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } -BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) +BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) { - return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}); + return std::make_shared(*this, metadata_snapshot); } -CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & context) +CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & /* context */) { - std::shared_lock lock(rwlock, getLockTimeout(context)); - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - + std::shared_lock lock(rwlock); return file_checker.check(); } void StorageTinyLog::truncate( const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { + std::unique_lock lock(rwlock); + disk->clearDirectory(table_path); files.clear(); @@ -507,6 +468,14 @@ void StorageTinyLog::truncate( addFiles(column.name, *column.type); } +void StorageTinyLog::drop() +{ + std::unique_lock lock(rwlock); + if (disk->exists(table_path)) + disk->removeRecursive(table_path); + files.clear(); +} + void registerStorageTinyLog(StorageFactory & factory) { diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 95b7d9f2941..dc6ff101503 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -43,6 +43,8 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) override; + void drop() override; + protected: StorageTinyLog( DiskPtr disk_, @@ -68,7 +70,7 @@ private: Files files; FileChecker file_checker; - mutable std::shared_timed_mutex rwlock; + mutable std::shared_mutex rwlock; Poco::Logger * log; diff --git a/tests/queries/0_stateless/01499_log_deadlock.reference b/tests/queries/0_stateless/01499_log_deadlock.reference deleted file mode 100644 index 166be640db5..00000000000 --- a/tests/queries/0_stateless/01499_log_deadlock.reference +++ /dev/null @@ -1,3 +0,0 @@ -6 -6 -6 diff --git a/tests/queries/0_stateless/01499_log_deadlock.sql b/tests/queries/0_stateless/01499_log_deadlock.sql deleted file mode 100644 index e98b37f2455..00000000000 --- a/tests/queries/0_stateless/01499_log_deadlock.sql +++ /dev/null @@ -1,26 +0,0 @@ -DROP TABLE IF EXISTS t; -CREATE TABLE t (x UInt8) ENGINE = TinyLog; - -INSERT INTO t VALUES (1), (2), (3); -INSERT INTO t SELECT * FROM t; -SELECT count() FROM t; - -DROP TABLE t; - - -CREATE TABLE t (x UInt8) ENGINE = Log; - -INSERT INTO t VALUES (1), (2), (3); -INSERT INTO t SELECT * FROM t; -SELECT count() FROM t; - -DROP TABLE t; - - -CREATE TABLE t (x UInt8) ENGINE = StripeLog; - -INSERT INTO t VALUES (1), (2), (3); -INSERT INTO t SELECT * FROM t; -SELECT count() FROM t; - -DROP TABLE t; diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference deleted file mode 100644 index 4bf85ae79f3..00000000000 --- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference +++ /dev/null @@ -1,6 +0,0 @@ -Testing TinyLog -Done TinyLog -Testing StripeLog -Done StripeLog -Testing Log -Done Log diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh deleted file mode 100755 index 29c5f868617..00000000000 --- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../shell_config.sh - - -function thread_create { - while true; do - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM - done -} - -function thread_drop { - while true; do - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM - done -} - -function thread_rename { - while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM - done -} - -function thread_select { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM - done -} - -function thread_insert { - while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM - done -} - -function thread_insert_select { - while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM - done -} - -export -f thread_create -export -f thread_drop -export -f thread_rename -export -f thread_select -export -f thread_insert -export -f thread_insert_select - - -# Do randomized queries and expect nothing extraordinary happens. - -function test_with_engine { - echo "Testing $1" - - timeout 10 bash -c "thread_create t1 $1" & - timeout 10 bash -c "thread_create t2 $1" & - timeout 10 bash -c 'thread_drop t1' & - timeout 10 bash -c 'thread_drop t2' & - timeout 10 bash -c 'thread_rename t1 t2' & - timeout 10 bash -c 'thread_rename t2 t1' & - timeout 10 bash -c 'thread_select t1' & - timeout 10 bash -c 'thread_select t2' & - timeout 10 bash -c 'thread_insert t1 5' & - timeout 10 bash -c 'thread_insert t2 10' & - timeout 10 bash -c 'thread_insert_select t1 t2' & - timeout 10 bash -c 'thread_insert_select t2 t1' & - - wait - echo "Done $1" -} - -test_with_engine TinyLog -test_with_engine StripeLog -test_with_engine Log diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.reference b/tests/queries/0_stateless/01505_log_distributed_deadlock.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql deleted file mode 100644 index 2b0b2b97188..00000000000 --- a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS t_local; -DROP TABLE IF EXISTS t_dist; - -create table t_local(a int) engine Log; -create table t_dist (a int) engine Distributed(test_shard_localhost, currentDatabase(), 't_local', cityHash64(a)); - -set insert_distributed_sync = 1; - -insert into t_dist values (1); - -DROP TABLE t_local; -DROP TABLE t_dist; diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6d1c6444d1b..69391ca9fd4 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -145,4 +145,3 @@ 01461_query_start_time_microseconds 01455_shard_leaf_max_rows_bytes_to_read 01505_distributed_local_type_conversion_enum -01505_log_distributed_deadlock From fb9d43692bcdd435377db0380e79dc33e47b6d7f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 03:10:37 +0300 Subject: [PATCH 339/390] Step 1: make adding hardware benchmark results more convenient --- .../results/{022_amd_epyc_7402p.json => amd_epyc_7402p.json} | 0 .../results/{043_amd_epyc_7502p.json => amd_epyc_7502p.json} | 0 .../results/{005_amd_epyc_7551.json => amd_epyc_7551.json} | 0 .../results/{052_amd_epyc_7642.json => amd_epyc_7642.json} | 0 .../results/{041_amd_epyc_7702.json => amd_epyc_7702.json} | 0 .../{038_amd_ryzen_9_3950x.json => amd_ryzen_9_3950x.json} | 0 .../results/{035_aws_a1_4xlarge.json => aws_a1_4xlarge.json} | 0 .../hardware/results/{049_aws_c5metal.json => aws_c5metal.json} | 0 .../results/{015_aws_i3_8xlarge.json => aws_i3_8xlarge.json} | 0 .../{017_aws_i3en_24xlarge.json => aws_i3en_24xlarge.json} | 0 .../{046_aws_lightsail_4vcpu.json => aws_lightsail_4vcpu.json} | 0 .../results/{051_aws_m5a_8xlarge.json => aws_m5a_8xlarge.json} | 0 .../{019_aws_m5ad_24xlarge.json => aws_m5ad_24xlarge.json} | 0 .../results/{016_aws_m5d_24xlarge.json => aws_m5d_24xlarge.json} | 0 .../results/{045_aws_m6g_16xlarge.json => aws_m6g_16xlarge.json} | 0 .../hardware/results/{014_azure_ds3v2.json => azure_ds3v2.json} | 0 .../hardware/results/{039_azure_e32s.json => azure_e32s.json} | 0 ...e_i5_3210M_lenovo_b580.json => core_i5_3210M_lenovo_b580.json} | 0 ...ore_i7_6770hq_intel_nuc.json => core_i7_6770hq_intel_nuc.json} | 0 ..._core_i7_8550u_lenovo_x1.json => core_i7_8550u_lenovo_x1.json} | 0 ...ore_i7_macbook_pro_2018.json => core_i7_macbook_pro_2018.json} | 0 .../hardware/results/{012_dell_r530.json => dell_r530.json} | 0 .../hardware/results/{047_dell_xps.json => dell_xps.json} | 0 ...18_huawei_taishan_2280_v2.json => huawei_taishan_2280_v2.json} | 0 .../hardware/results/{037_pinebook_pro.json => pinebook_pro.json} | 0 .../hardware/results/{048_pixel_3a.json => pixel_3a.json} | 0 ...{024_selectel_cloud_16vcpu.json => selectel_cloud_16vcpu.json} | 0 .../hardware/results/{008_skylake_kvm.json => skylake_kvm.json} | 0 .../hardware/results/{013_xeon_2176g.json => xeon_2176g.json} | 0 .../hardware/results/{021_xeon_e5645.json => xeon_e5645.json} | 0 .../results/{023_xeon_e5_1650v3.json => xeon_e5_1650v3.json} | 0 .../results/{010_xeon_e5_2640v4.json => xeon_e5_2640v4.json} | 0 .../hardware/results/{007_xeon_e5_2650.json => xeon_e5_2650.json} | 0 .../results/{050_xeon_e5_2650l_v3.json => xeon_e5_2650l_v3.json} | 0 .../results/{001_xeon_gold_6230.json => xeon_gold_6230.json} | 0 .../results/{044_xeon_silver_4114.json => xeon_silver_4114.json} | 0 .../hardware/results/{006_xeon_sp_gold.json => xeon_sp_gold.json} | 0 .../hardware/results/{036_xeon_x5675.json => xeon_x5675.json} | 0 ...d_broadwell_4_vcpu.json => yandex_cloud_broadwell_4_vcpu.json} | 0 ...ade_lake_4_vcpu.json => yandex_cloud_cascade_lake_4_vcpu.json} | 0 ...e_lake_64_vcpu.json => yandex_cloud_cascade_lake_64_vcpu.json} | 0 ..._s3_3xlarge.json => yandex_managed_clickhouse_s3_3xlarge.json} | 0 42 files changed, 0 insertions(+), 0 deletions(-) rename website/benchmark/hardware/results/{022_amd_epyc_7402p.json => amd_epyc_7402p.json} (100%) rename website/benchmark/hardware/results/{043_amd_epyc_7502p.json => amd_epyc_7502p.json} (100%) rename website/benchmark/hardware/results/{005_amd_epyc_7551.json => amd_epyc_7551.json} (100%) rename website/benchmark/hardware/results/{052_amd_epyc_7642.json => amd_epyc_7642.json} (100%) rename website/benchmark/hardware/results/{041_amd_epyc_7702.json => amd_epyc_7702.json} (100%) rename website/benchmark/hardware/results/{038_amd_ryzen_9_3950x.json => amd_ryzen_9_3950x.json} (100%) rename website/benchmark/hardware/results/{035_aws_a1_4xlarge.json => aws_a1_4xlarge.json} (100%) rename website/benchmark/hardware/results/{049_aws_c5metal.json => aws_c5metal.json} (100%) rename website/benchmark/hardware/results/{015_aws_i3_8xlarge.json => aws_i3_8xlarge.json} (100%) rename website/benchmark/hardware/results/{017_aws_i3en_24xlarge.json => aws_i3en_24xlarge.json} (100%) rename website/benchmark/hardware/results/{046_aws_lightsail_4vcpu.json => aws_lightsail_4vcpu.json} (100%) rename website/benchmark/hardware/results/{051_aws_m5a_8xlarge.json => aws_m5a_8xlarge.json} (100%) rename website/benchmark/hardware/results/{019_aws_m5ad_24xlarge.json => aws_m5ad_24xlarge.json} (100%) rename website/benchmark/hardware/results/{016_aws_m5d_24xlarge.json => aws_m5d_24xlarge.json} (100%) rename website/benchmark/hardware/results/{045_aws_m6g_16xlarge.json => aws_m6g_16xlarge.json} (100%) rename website/benchmark/hardware/results/{014_azure_ds3v2.json => azure_ds3v2.json} (100%) rename website/benchmark/hardware/results/{039_azure_e32s.json => azure_e32s.json} (100%) rename website/benchmark/hardware/results/{009_core_i5_3210M_lenovo_b580.json => core_i5_3210M_lenovo_b580.json} (100%) rename website/benchmark/hardware/results/{042_core_i7_6770hq_intel_nuc.json => core_i7_6770hq_intel_nuc.json} (100%) rename website/benchmark/hardware/results/{020_core_i7_8550u_lenovo_x1.json => core_i7_8550u_lenovo_x1.json} (100%) rename website/benchmark/hardware/results/{040_core_i7_macbook_pro_2018.json => core_i7_macbook_pro_2018.json} (100%) rename website/benchmark/hardware/results/{012_dell_r530.json => dell_r530.json} (100%) rename website/benchmark/hardware/results/{047_dell_xps.json => dell_xps.json} (100%) rename website/benchmark/hardware/results/{018_huawei_taishan_2280_v2.json => huawei_taishan_2280_v2.json} (100%) rename website/benchmark/hardware/results/{037_pinebook_pro.json => pinebook_pro.json} (100%) rename website/benchmark/hardware/results/{048_pixel_3a.json => pixel_3a.json} (100%) rename website/benchmark/hardware/results/{024_selectel_cloud_16vcpu.json => selectel_cloud_16vcpu.json} (100%) rename website/benchmark/hardware/results/{008_skylake_kvm.json => skylake_kvm.json} (100%) rename website/benchmark/hardware/results/{013_xeon_2176g.json => xeon_2176g.json} (100%) rename website/benchmark/hardware/results/{021_xeon_e5645.json => xeon_e5645.json} (100%) rename website/benchmark/hardware/results/{023_xeon_e5_1650v3.json => xeon_e5_1650v3.json} (100%) rename website/benchmark/hardware/results/{010_xeon_e5_2640v4.json => xeon_e5_2640v4.json} (100%) rename website/benchmark/hardware/results/{007_xeon_e5_2650.json => xeon_e5_2650.json} (100%) rename website/benchmark/hardware/results/{050_xeon_e5_2650l_v3.json => xeon_e5_2650l_v3.json} (100%) rename website/benchmark/hardware/results/{001_xeon_gold_6230.json => xeon_gold_6230.json} (100%) rename website/benchmark/hardware/results/{044_xeon_silver_4114.json => xeon_silver_4114.json} (100%) rename website/benchmark/hardware/results/{006_xeon_sp_gold.json => xeon_sp_gold.json} (100%) rename website/benchmark/hardware/results/{036_xeon_x5675.json => xeon_x5675.json} (100%) rename website/benchmark/hardware/results/{004_yandex_cloud_broadwell_4_vcpu.json => yandex_cloud_broadwell_4_vcpu.json} (100%) rename website/benchmark/hardware/results/{003_yandex_cloud_cascade_lake_4_vcpu.json => yandex_cloud_cascade_lake_4_vcpu.json} (100%) rename website/benchmark/hardware/results/{002_yandex_cloud_cascade_lake_64_vcpu.json => yandex_cloud_cascade_lake_64_vcpu.json} (100%) rename website/benchmark/hardware/results/{011_yandex_managed_clickhouse_s3_3xlarge.json => yandex_managed_clickhouse_s3_3xlarge.json} (100%) diff --git a/website/benchmark/hardware/results/022_amd_epyc_7402p.json b/website/benchmark/hardware/results/amd_epyc_7402p.json similarity index 100% rename from website/benchmark/hardware/results/022_amd_epyc_7402p.json rename to website/benchmark/hardware/results/amd_epyc_7402p.json diff --git a/website/benchmark/hardware/results/043_amd_epyc_7502p.json b/website/benchmark/hardware/results/amd_epyc_7502p.json similarity index 100% rename from website/benchmark/hardware/results/043_amd_epyc_7502p.json rename to website/benchmark/hardware/results/amd_epyc_7502p.json diff --git a/website/benchmark/hardware/results/005_amd_epyc_7551.json b/website/benchmark/hardware/results/amd_epyc_7551.json similarity index 100% rename from website/benchmark/hardware/results/005_amd_epyc_7551.json rename to website/benchmark/hardware/results/amd_epyc_7551.json diff --git a/website/benchmark/hardware/results/052_amd_epyc_7642.json b/website/benchmark/hardware/results/amd_epyc_7642.json similarity index 100% rename from website/benchmark/hardware/results/052_amd_epyc_7642.json rename to website/benchmark/hardware/results/amd_epyc_7642.json diff --git a/website/benchmark/hardware/results/041_amd_epyc_7702.json b/website/benchmark/hardware/results/amd_epyc_7702.json similarity index 100% rename from website/benchmark/hardware/results/041_amd_epyc_7702.json rename to website/benchmark/hardware/results/amd_epyc_7702.json diff --git a/website/benchmark/hardware/results/038_amd_ryzen_9_3950x.json b/website/benchmark/hardware/results/amd_ryzen_9_3950x.json similarity index 100% rename from website/benchmark/hardware/results/038_amd_ryzen_9_3950x.json rename to website/benchmark/hardware/results/amd_ryzen_9_3950x.json diff --git a/website/benchmark/hardware/results/035_aws_a1_4xlarge.json b/website/benchmark/hardware/results/aws_a1_4xlarge.json similarity index 100% rename from website/benchmark/hardware/results/035_aws_a1_4xlarge.json rename to website/benchmark/hardware/results/aws_a1_4xlarge.json diff --git a/website/benchmark/hardware/results/049_aws_c5metal.json b/website/benchmark/hardware/results/aws_c5metal.json similarity index 100% rename from website/benchmark/hardware/results/049_aws_c5metal.json rename to website/benchmark/hardware/results/aws_c5metal.json diff --git a/website/benchmark/hardware/results/015_aws_i3_8xlarge.json b/website/benchmark/hardware/results/aws_i3_8xlarge.json similarity index 100% rename from website/benchmark/hardware/results/015_aws_i3_8xlarge.json rename to website/benchmark/hardware/results/aws_i3_8xlarge.json diff --git a/website/benchmark/hardware/results/017_aws_i3en_24xlarge.json b/website/benchmark/hardware/results/aws_i3en_24xlarge.json similarity index 100% rename from website/benchmark/hardware/results/017_aws_i3en_24xlarge.json rename to website/benchmark/hardware/results/aws_i3en_24xlarge.json diff --git a/website/benchmark/hardware/results/046_aws_lightsail_4vcpu.json b/website/benchmark/hardware/results/aws_lightsail_4vcpu.json similarity index 100% rename from website/benchmark/hardware/results/046_aws_lightsail_4vcpu.json rename to website/benchmark/hardware/results/aws_lightsail_4vcpu.json diff --git a/website/benchmark/hardware/results/051_aws_m5a_8xlarge.json b/website/benchmark/hardware/results/aws_m5a_8xlarge.json similarity index 100% rename from website/benchmark/hardware/results/051_aws_m5a_8xlarge.json rename to website/benchmark/hardware/results/aws_m5a_8xlarge.json diff --git a/website/benchmark/hardware/results/019_aws_m5ad_24xlarge.json b/website/benchmark/hardware/results/aws_m5ad_24xlarge.json similarity index 100% rename from website/benchmark/hardware/results/019_aws_m5ad_24xlarge.json rename to website/benchmark/hardware/results/aws_m5ad_24xlarge.json diff --git a/website/benchmark/hardware/results/016_aws_m5d_24xlarge.json b/website/benchmark/hardware/results/aws_m5d_24xlarge.json similarity index 100% rename from website/benchmark/hardware/results/016_aws_m5d_24xlarge.json rename to website/benchmark/hardware/results/aws_m5d_24xlarge.json diff --git a/website/benchmark/hardware/results/045_aws_m6g_16xlarge.json b/website/benchmark/hardware/results/aws_m6g_16xlarge.json similarity index 100% rename from website/benchmark/hardware/results/045_aws_m6g_16xlarge.json rename to website/benchmark/hardware/results/aws_m6g_16xlarge.json diff --git a/website/benchmark/hardware/results/014_azure_ds3v2.json b/website/benchmark/hardware/results/azure_ds3v2.json similarity index 100% rename from website/benchmark/hardware/results/014_azure_ds3v2.json rename to website/benchmark/hardware/results/azure_ds3v2.json diff --git a/website/benchmark/hardware/results/039_azure_e32s.json b/website/benchmark/hardware/results/azure_e32s.json similarity index 100% rename from website/benchmark/hardware/results/039_azure_e32s.json rename to website/benchmark/hardware/results/azure_e32s.json diff --git a/website/benchmark/hardware/results/009_core_i5_3210M_lenovo_b580.json b/website/benchmark/hardware/results/core_i5_3210M_lenovo_b580.json similarity index 100% rename from website/benchmark/hardware/results/009_core_i5_3210M_lenovo_b580.json rename to website/benchmark/hardware/results/core_i5_3210M_lenovo_b580.json diff --git a/website/benchmark/hardware/results/042_core_i7_6770hq_intel_nuc.json b/website/benchmark/hardware/results/core_i7_6770hq_intel_nuc.json similarity index 100% rename from website/benchmark/hardware/results/042_core_i7_6770hq_intel_nuc.json rename to website/benchmark/hardware/results/core_i7_6770hq_intel_nuc.json diff --git a/website/benchmark/hardware/results/020_core_i7_8550u_lenovo_x1.json b/website/benchmark/hardware/results/core_i7_8550u_lenovo_x1.json similarity index 100% rename from website/benchmark/hardware/results/020_core_i7_8550u_lenovo_x1.json rename to website/benchmark/hardware/results/core_i7_8550u_lenovo_x1.json diff --git a/website/benchmark/hardware/results/040_core_i7_macbook_pro_2018.json b/website/benchmark/hardware/results/core_i7_macbook_pro_2018.json similarity index 100% rename from website/benchmark/hardware/results/040_core_i7_macbook_pro_2018.json rename to website/benchmark/hardware/results/core_i7_macbook_pro_2018.json diff --git a/website/benchmark/hardware/results/012_dell_r530.json b/website/benchmark/hardware/results/dell_r530.json similarity index 100% rename from website/benchmark/hardware/results/012_dell_r530.json rename to website/benchmark/hardware/results/dell_r530.json diff --git a/website/benchmark/hardware/results/047_dell_xps.json b/website/benchmark/hardware/results/dell_xps.json similarity index 100% rename from website/benchmark/hardware/results/047_dell_xps.json rename to website/benchmark/hardware/results/dell_xps.json diff --git a/website/benchmark/hardware/results/018_huawei_taishan_2280_v2.json b/website/benchmark/hardware/results/huawei_taishan_2280_v2.json similarity index 100% rename from website/benchmark/hardware/results/018_huawei_taishan_2280_v2.json rename to website/benchmark/hardware/results/huawei_taishan_2280_v2.json diff --git a/website/benchmark/hardware/results/037_pinebook_pro.json b/website/benchmark/hardware/results/pinebook_pro.json similarity index 100% rename from website/benchmark/hardware/results/037_pinebook_pro.json rename to website/benchmark/hardware/results/pinebook_pro.json diff --git a/website/benchmark/hardware/results/048_pixel_3a.json b/website/benchmark/hardware/results/pixel_3a.json similarity index 100% rename from website/benchmark/hardware/results/048_pixel_3a.json rename to website/benchmark/hardware/results/pixel_3a.json diff --git a/website/benchmark/hardware/results/024_selectel_cloud_16vcpu.json b/website/benchmark/hardware/results/selectel_cloud_16vcpu.json similarity index 100% rename from website/benchmark/hardware/results/024_selectel_cloud_16vcpu.json rename to website/benchmark/hardware/results/selectel_cloud_16vcpu.json diff --git a/website/benchmark/hardware/results/008_skylake_kvm.json b/website/benchmark/hardware/results/skylake_kvm.json similarity index 100% rename from website/benchmark/hardware/results/008_skylake_kvm.json rename to website/benchmark/hardware/results/skylake_kvm.json diff --git a/website/benchmark/hardware/results/013_xeon_2176g.json b/website/benchmark/hardware/results/xeon_2176g.json similarity index 100% rename from website/benchmark/hardware/results/013_xeon_2176g.json rename to website/benchmark/hardware/results/xeon_2176g.json diff --git a/website/benchmark/hardware/results/021_xeon_e5645.json b/website/benchmark/hardware/results/xeon_e5645.json similarity index 100% rename from website/benchmark/hardware/results/021_xeon_e5645.json rename to website/benchmark/hardware/results/xeon_e5645.json diff --git a/website/benchmark/hardware/results/023_xeon_e5_1650v3.json b/website/benchmark/hardware/results/xeon_e5_1650v3.json similarity index 100% rename from website/benchmark/hardware/results/023_xeon_e5_1650v3.json rename to website/benchmark/hardware/results/xeon_e5_1650v3.json diff --git a/website/benchmark/hardware/results/010_xeon_e5_2640v4.json b/website/benchmark/hardware/results/xeon_e5_2640v4.json similarity index 100% rename from website/benchmark/hardware/results/010_xeon_e5_2640v4.json rename to website/benchmark/hardware/results/xeon_e5_2640v4.json diff --git a/website/benchmark/hardware/results/007_xeon_e5_2650.json b/website/benchmark/hardware/results/xeon_e5_2650.json similarity index 100% rename from website/benchmark/hardware/results/007_xeon_e5_2650.json rename to website/benchmark/hardware/results/xeon_e5_2650.json diff --git a/website/benchmark/hardware/results/050_xeon_e5_2650l_v3.json b/website/benchmark/hardware/results/xeon_e5_2650l_v3.json similarity index 100% rename from website/benchmark/hardware/results/050_xeon_e5_2650l_v3.json rename to website/benchmark/hardware/results/xeon_e5_2650l_v3.json diff --git a/website/benchmark/hardware/results/001_xeon_gold_6230.json b/website/benchmark/hardware/results/xeon_gold_6230.json similarity index 100% rename from website/benchmark/hardware/results/001_xeon_gold_6230.json rename to website/benchmark/hardware/results/xeon_gold_6230.json diff --git a/website/benchmark/hardware/results/044_xeon_silver_4114.json b/website/benchmark/hardware/results/xeon_silver_4114.json similarity index 100% rename from website/benchmark/hardware/results/044_xeon_silver_4114.json rename to website/benchmark/hardware/results/xeon_silver_4114.json diff --git a/website/benchmark/hardware/results/006_xeon_sp_gold.json b/website/benchmark/hardware/results/xeon_sp_gold.json similarity index 100% rename from website/benchmark/hardware/results/006_xeon_sp_gold.json rename to website/benchmark/hardware/results/xeon_sp_gold.json diff --git a/website/benchmark/hardware/results/036_xeon_x5675.json b/website/benchmark/hardware/results/xeon_x5675.json similarity index 100% rename from website/benchmark/hardware/results/036_xeon_x5675.json rename to website/benchmark/hardware/results/xeon_x5675.json diff --git a/website/benchmark/hardware/results/004_yandex_cloud_broadwell_4_vcpu.json b/website/benchmark/hardware/results/yandex_cloud_broadwell_4_vcpu.json similarity index 100% rename from website/benchmark/hardware/results/004_yandex_cloud_broadwell_4_vcpu.json rename to website/benchmark/hardware/results/yandex_cloud_broadwell_4_vcpu.json diff --git a/website/benchmark/hardware/results/003_yandex_cloud_cascade_lake_4_vcpu.json b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_4_vcpu.json similarity index 100% rename from website/benchmark/hardware/results/003_yandex_cloud_cascade_lake_4_vcpu.json rename to website/benchmark/hardware/results/yandex_cloud_cascade_lake_4_vcpu.json diff --git a/website/benchmark/hardware/results/002_yandex_cloud_cascade_lake_64_vcpu.json b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_64_vcpu.json similarity index 100% rename from website/benchmark/hardware/results/002_yandex_cloud_cascade_lake_64_vcpu.json rename to website/benchmark/hardware/results/yandex_cloud_cascade_lake_64_vcpu.json diff --git a/website/benchmark/hardware/results/011_yandex_managed_clickhouse_s3_3xlarge.json b/website/benchmark/hardware/results/yandex_managed_clickhouse_s3_3xlarge.json similarity index 100% rename from website/benchmark/hardware/results/011_yandex_managed_clickhouse_s3_3xlarge.json rename to website/benchmark/hardware/results/yandex_managed_clickhouse_s3_3xlarge.json From 8e986f81b441822884c801e0763798dc03ee710f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 03:24:24 +0300 Subject: [PATCH 340/390] Added new results --- ...{aws_c5metal.json => aws_c5metal_100.json} | 2 +- .../hardware/results/aws_c5metal_300.json | 54 ++++++++++++++++++ .../hardware/results/aws_c6metal.json | 54 ++++++++++++++++++ .../yandex_cloud_cascade_lake_32_vcpu.json | 55 +++++++++++++++++++ .../yandex_cloud_cascade_lake_80_vcpu.json | 55 +++++++++++++++++++ 5 files changed, 219 insertions(+), 1 deletion(-) rename website/benchmark/hardware/results/{aws_c5metal.json => aws_c5metal_100.json} (97%) create mode 100644 website/benchmark/hardware/results/aws_c5metal_300.json create mode 100644 website/benchmark/hardware/results/aws_c6metal.json create mode 100644 website/benchmark/hardware/results/yandex_cloud_cascade_lake_32_vcpu.json create mode 100644 website/benchmark/hardware/results/yandex_cloud_cascade_lake_80_vcpu.json diff --git a/website/benchmark/hardware/results/aws_c5metal.json b/website/benchmark/hardware/results/aws_c5metal_100.json similarity index 97% rename from website/benchmark/hardware/results/aws_c5metal.json rename to website/benchmark/hardware/results/aws_c5metal_100.json index 9d933500ad1..4bb0a1f1f52 100644 --- a/website/benchmark/hardware/results/aws_c5metal.json +++ b/website/benchmark/hardware/results/aws_c5metal_100.json @@ -1,6 +1,6 @@ [ { - "system": "AWS c5.metal", + "system": "AWS c5.metal 100GB", "system_full": "AWS c5.metal 96vCPU 192GiB 100GB SSD", "time": "2020-01-17 00:00:00", "kind": "cloud", diff --git a/website/benchmark/hardware/results/aws_c5metal_300.json b/website/benchmark/hardware/results/aws_c5metal_300.json new file mode 100644 index 00000000000..87435f6fb45 --- /dev/null +++ b/website/benchmark/hardware/results/aws_c5metal_300.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS c5.metal 300GB", + "system_full": "AWS c5.metal 96vCPU 192GiB 300GB SSD", + "time": "2020-09-23 00:00:00", + "kind": "cloud", + "result": + [ +[0.012, 0.002, 0.002], +[0.066, 0.018, 0.018], +[0.066, 0.028, 0.027], +[0.186, 0.033, 0.031], +[0.362, 0.095, 0.093], +[1.092, 0.141, 0.142], +[0.035, 0.020, 0.021], +[0.023, 0.018, 0.018], +[0.303, 0.176, 0.181], +[0.817, 0.198, 0.198], +[0.322, 0.091, 0.092], +[0.600, 0.098, 0.098], +[1.059, 0.265, 0.253], +[1.542, 0.318, 0.310], +[0.682, 0.286, 0.283], +[0.372, 0.320, 0.295], +[1.610, 0.832, 0.750], +[1.301, 0.492, 0.458], +[3.446, 1.361, 1.330], +[0.189, 0.050, 0.035], +[9.246, 0.338, 0.265], +[10.163, 0.277, 0.249], +[19.616, 0.663, 0.639], +[20.068, 0.418, 0.367], +[1.812, 0.097, 0.093], +[0.976, 0.090, 0.083], +[2.458, 0.097, 0.095], +[9.397, 0.344, 0.323], +[7.320, 0.415, 0.413], +[0.780, 0.753, 0.748], +[1.328, 0.226, 0.223], +[4.643, 0.339, 0.329], +[4.136, 2.049, 2.021], +[9.213, 1.080, 0.923], +[9.192, 1.019, 0.959], +[0.410, 0.360, 0.378], +[0.244, 0.155, 0.163], +[0.102, 0.077, 0.071], +[0.045, 0.055, 0.049], +[0.459, 0.318, 0.316], +[0.069, 0.033, 0.026], +[0.035, 0.027, 0.020], +[0.019, 0.009, 0.010] + ] + } +] diff --git a/website/benchmark/hardware/results/aws_c6metal.json b/website/benchmark/hardware/results/aws_c6metal.json new file mode 100644 index 00000000000..83e75506ad9 --- /dev/null +++ b/website/benchmark/hardware/results/aws_c6metal.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS c6.metal (Graviton 2)", + "system_full": "AWS c6.metal (Graviton 2) 64 CPU 128GiB 2x1.7TB local SSD md-RAID-0", + "time": "2020-09-23 00:00:00", + "kind": "cloud", + "result": + [ +[0.004, 0.003, 0.001], +[0.085, 0.030, 0.032], +[0.029, 0.028, 0.026], +[0.047, 0.068, 0.070], +[0.090, 0.075, 0.079], +[0.140, 0.126, 0.124], +[0.018, 0.013, 0.012], +[0.032, 0.021, 0.032], +[0.154, 0.139, 0.138], +[0.204, 0.155, 0.156], +[0.101, 0.091, 0.090], +[0.104, 0.104, 0.100], +[0.223, 0.203, 0.203], +[0.273, 0.255, 0.253], +[0.232, 0.212, 0.213], +[0.230, 0.223, 0.223], +[0.506, 0.484, 0.483], +[0.334, 0.330, 0.316], +[1.139, 1.085, 1.088], +[0.065, 0.077, 0.054], +[0.484, 0.315, 0.315], +[0.545, 0.295, 0.291], +[0.980, 0.661, 1.476], +[1.415, 1.101, 0.675], +[0.150, 0.086, 0.085], +[0.094, 0.077, 0.078], +[0.150, 0.087, 0.086], +[0.478, 0.348, 0.346], +[0.424, 0.403, 0.399], +[1.435, 1.388, 1.417], +[0.215, 0.178, 0.178], +[0.378, 0.294, 0.289], +[1.669, 1.590, 1.596], +[1.105, 1.007, 1.010], +[1.074, 1.041, 1.014], +[0.339, 0.323, 0.323], +[0.210, 0.199, 0.204], +[0.096, 0.091, 0.092], +[0.084, 0.080, 0.079], +[0.425, 0.405, 0.423], +[0.034, 0.025, 0.022], +[0.022, 0.019, 0.018], +[0.007, 0.007, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/yandex_cloud_cascade_lake_32_vcpu.json b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_32_vcpu.json new file mode 100644 index 00000000000..5d2927c224d --- /dev/null +++ b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_32_vcpu.json @@ -0,0 +1,55 @@ +[ + { + "system": "Yandex Cloud 32vCPU", + "system_full": "Yandex Cloud Cascade Lake, 32 vCPU, 128 GB RAM, 300 GB SSD", + "cpu_vendor": "Intel", + "time": "2020-09-23 00:00:00", + "kind": "cloud", + "result": + [ +[0.021, 0.001, 0.001], +[0.051, 0.011, 0.010], +[0.396, 0.025, 0.025], +[1.400, 0.035, 0.033], +[1.413, 0.095, 0.098], +[2.272, 0.222, 0.208], +[0.042, 0.014, 0.014], +[0.024, 0.011, 0.010], +[1.948, 0.311, 0.303], +[2.267, 0.379, 0.348], +[1.498, 0.138, 0.135], +[1.563, 0.164, 0.155], +[2.435, 0.544, 0.516], +[3.937, 0.661, 0.659], +[2.724, 0.727, 0.642], +[1.795, 0.683, 0.641], +[4.668, 1.682, 1.643], +[3.802, 1.051, 0.895], +[8.297, 3.835, 4.592], +[1.427, 0.100, 0.033], +[16.816, 0.652, 0.547], +[19.159, 0.650, 0.532], +[35.374, 1.538, 1.311], +[32.736, 0.854, 0.699], +[4.767, 0.203, 0.184], +[2.249, 0.166, 0.158], +[4.759, 0.207, 0.189], +[16.826, 0.584, 0.529], +[14.308, 0.920, 0.789], +[1.137, 1.041, 0.992], +[3.967, 0.545, 0.555], +[9.196, 0.872, 0.789], +[9.554, 5.501, 5.694], +[17.810, 2.712, 2.329], +[17.726, 2.653, 2.793], +[1.260, 0.955, 0.978], +[0.260, 0.171, 0.164], +[0.092, 0.065, 0.069], +[0.046, 0.041, 0.037], +[0.475, 0.391, 0.383], +[0.066, 0.021, 0.019], +[0.023, 0.024, 0.011], +[0.022, 0.005, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/yandex_cloud_cascade_lake_80_vcpu.json b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_80_vcpu.json new file mode 100644 index 00000000000..565a5bd41c2 --- /dev/null +++ b/website/benchmark/hardware/results/yandex_cloud_cascade_lake_80_vcpu.json @@ -0,0 +1,55 @@ +[ + { + "system": "Yandex Cloud 80vCPU", + "system_full": "Yandex Cloud Cascade Lake, 80 vCPU, 160 GB RAM, 4TB SSD", + "cpu_vendor": "Intel", + "time": "2020-09-23 00:00:00", + "kind": "cloud", + "result": + [ +[0.024, 0.002, 0.002], +[0.067, 0.012, 0.012], +[0.104, 0.017, 0.017], +[0.411, 0.020, 0.021], +[0.577, 0.069, 0.068], +[0.739, 0.123, 0.122], +[0.038, 0.015, 0.014], +[0.024, 0.012, 0.012], +[0.625, 0.169, 0.168], +[0.748, 0.216, 0.207], +[0.471, 0.089, 0.082], +[0.487, 0.092, 0.087], +[0.818, 0.256, 0.245], +[1.324, 0.352, 0.352], +[0.927, 0.333, 0.319], +[0.642, 0.376, 0.377], +[1.686, 0.983, 0.959], +[1.290, 0.588, 0.582], +[3.105, 1.793, 1.818], +[0.426, 0.031, 0.034], +[5.559, 0.415, 0.344], +[6.343, 0.435, 0.405], +[11.779, 1.151, 1.101], +[11.851, 0.537, 0.509], +[1.530, 0.125, 0.126], +[0.695, 0.103, 0.103], +[1.531, 0.127, 0.119], +[5.576, 0.541, 0.496], +[4.718, 0.740, 0.719], +[1.429, 1.467, 1.500], +[1.309, 0.335, 0.322], +[3.138, 0.505, 0.518], +[5.481, 3.475, 3.512], +[6.330, 1.877, 1.818], +[6.238, 1.843, 1.813], +[0.660, 0.626, 0.603], +[0.251, 0.152, 0.151], +[0.090, 0.058, 0.059], +[0.041, 0.038, 0.034], +[0.470, 0.376, 0.385], +[0.076, 0.015, 0.018], +[0.030, 0.018, 0.010], +[0.024, 0.006, 0.005] + ] + } +] From 4e0378a521f6bb606723109b445e6ddd27bc48f8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 03:29:42 +0300 Subject: [PATCH 341/390] Adjustments --- benchmark/hardware.sh | 8 +++++--- website/benchmark/benchmark.js | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 693a69deb1f..cfcbe6ce24c 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -60,7 +60,7 @@ if [[ ! -d data ]]; then if [[ ! -f $DATASET ]]; then $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET" fi - + tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET fi @@ -105,8 +105,10 @@ echo echo "Benchmark complete. System info:" echo -echo '----Version and build id--------' -./clickhouse local --query "SELECT version(), buildId()" +echo '----Version, build id-----------' +./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())" +./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" +./clickhouse local --query "SELECT format('Current time: {}', toString(now('UTC')))" echo '----CPU-------------------------' lscpu echo '----Block Devices---------------' diff --git a/website/benchmark/benchmark.js b/website/benchmark/benchmark.js index 6113864d4d1..8fb2693aa97 100644 --- a/website/benchmark/benchmark.js +++ b/website/benchmark/benchmark.js @@ -403,7 +403,7 @@ function generate_diagram() { var table_row = ""; table_row += ""; - table_row += "
' + str(x) + '{value}
Date: Fri, 25 Sep 2020 03:36:06 +0300 Subject: [PATCH 342/390] Adjustments --- benchmark/hardware.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index cfcbe6ce24c..040f9169101 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -107,8 +107,8 @@ echo echo '----Version, build id-----------' ./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())" -./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" -./clickhouse local --query "SELECT format('Current time: {}', toString(now('UTC')))" +./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw +./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' lscpu echo '----Block Devices---------------' From 092ea18dd4d81fd4e3b89b748c85c33092e87f52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 03:48:18 +0300 Subject: [PATCH 343/390] Adjustments --- website/benchmark/hardware/results/aws_m6g_16xlarge.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/benchmark/hardware/results/aws_m6g_16xlarge.json b/website/benchmark/hardware/results/aws_m6g_16xlarge.json index 323fd2cc50c..a0d15a0d384 100644 --- a/website/benchmark/hardware/results/aws_m6g_16xlarge.json +++ b/website/benchmark/hardware/results/aws_m6g_16xlarge.json @@ -1,7 +1,7 @@ [ { - "system": "AWS m6g.16xlarge", - "system_full": "AWS m6g.16xlarge (Graviton2) 64 vCPU, 256 GiB RAM, EBS", + "system": "AWS m6g.16xlarge (Graviton 2)", + "system_full": "AWS m6g.16xlarge (Graviton 2) 64 vCPU, 256 GiB RAM, EBS", "time": "2020-02-13 00:00:00", "kind": "cloud", "result": From 1bd6fb39f217a6464476c2174abeae416960d2f1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 03:55:57 +0300 Subject: [PATCH 344/390] Added a comment --- benchmark/hardware.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 040f9169101..90e0e7dadfa 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -14,6 +14,8 @@ TRIES=3 AMD64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" AARCH64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse" +# Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'. + FASTER_DOWNLOAD=wget if command -v axel >/dev/null; then FASTER_DOWNLOAD=axel From fdf860adde2dc20fcedc1ffc0f78ed9d82f8ee57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 04:03:17 +0300 Subject: [PATCH 345/390] More handy --- benchmark/hardware.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 90e0e7dadfa..bb3492844a7 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -66,6 +66,8 @@ if [[ ! -d data ]]; then tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET fi +uptime + echo "Starting clickhouse-server" ./clickhouse server > server.log 2>&1 & From 8b55224350ea0c93fcfb2d0fd9830a00dfba6a66 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 04:06:21 +0300 Subject: [PATCH 346/390] More handy --- benchmark/hardware.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index bb3492844a7..286482d9132 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -114,6 +114,7 @@ echo '----Version, build id-----------' ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' +cat /proc/cpuinfo | grep -i -F 'model name' | uniq lscpu echo '----Block Devices---------------' lsblk From 0e7f576bc1256faf86bddf97c36b465a5e738a8f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 04:21:03 +0300 Subject: [PATCH 347/390] More results --- .../hardware/results/xeon_e5_2650_4hdd.json | 54 +++++++++++++++++++ ...on_e5_2650.json => xeon_e5_2650_8hdd.json} | 0 2 files changed, 54 insertions(+) create mode 100644 website/benchmark/hardware/results/xeon_e5_2650_4hdd.json rename website/benchmark/hardware/results/{xeon_e5_2650.json => xeon_e5_2650_8hdd.json} (100%) diff --git a/website/benchmark/hardware/results/xeon_e5_2650_4hdd.json b/website/benchmark/hardware/results/xeon_e5_2650_4hdd.json new file mode 100644 index 00000000000..478229badcc --- /dev/null +++ b/website/benchmark/hardware/results/xeon_e5_2650_4hdd.json @@ -0,0 +1,54 @@ +[ + { + "system": "Xeon E5-2650", + "system_full": "Xeon E5-2650 v2 @ 2.60GHz, 2 sockets, 16 threads, 4xHDD RAID-10", + "time": "2020-09-25 00:00:00", + "kind": "server", + "result": + [ +[0.040, 0.002, 0.002], +[0.698, 0.014, 0.013], +[0.533, 0.030, 0.030], +[0.700, 0.043, 0.046], +[0.749, 0.108, 0.102], +[1.350, 0.221, 0.259], +[0.168, 0.020, 0.020], +[0.096, 0.013, 0.013], +[1.132, 0.406, 0.386], +[1.279, 0.426, 0.440], +[0.842, 0.153, 0.146], +[1.042, 0.186, 0.182], +[1.149, 0.536, 0.533], +[1.734, 0.688, 0.683], +[1.481, 0.688, 0.651], +[1.100, 0.709, 0.700], +[2.367, 1.668, 1.682], +[1.687, 1.013, 0.988], +[4.768, 3.647, 3.783], +[0.599, 0.055, 0.040], +[5.530, 0.646, 0.622], +[6.658, 0.671, 0.648], +[11.795, 1.645, 1.574], +[19.248, 1.168, 0.906], +[1.826, 0.224, 0.232], +[0.964, 0.189, 0.187], +[2.058, 0.234, 0.215], +[5.811, 0.758, 0.704], +[4.805, 1.014, 0.995], +[2.272, 2.035, 1.838], +[1.827, 0.546, 0.547], +[3.643, 0.863, 0.834], +[5.816, 5.069, 5.168], +[6.585, 2.655, 2.756], +[6.949, 2.681, 2.795], +[1.325, 1.090, 1.072], +[0.460, 0.183, 0.179], +[1.000, 0.087, 0.091], +[0.142, 0.051, 0.038], +[0.808, 0.392, 0.391], +[0.256, 0.021, 0.015], +[0.132, 0.038, 0.012], +[0.054, 0.006, 0.006] + ] + } +] diff --git a/website/benchmark/hardware/results/xeon_e5_2650.json b/website/benchmark/hardware/results/xeon_e5_2650_8hdd.json similarity index 100% rename from website/benchmark/hardware/results/xeon_e5_2650.json rename to website/benchmark/hardware/results/xeon_e5_2650_8hdd.json From f5eead5b2aba0e4c719440c03b73bf0bc5caa32a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 04:45:47 +0300 Subject: [PATCH 348/390] Enable embedded configs for AArch64 --- programs/server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 5500a4680b7..b3dcf1955fe 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -3,7 +3,7 @@ set(CLICKHOUSE_SERVER_SOURCES Server.cpp ) -if (OS_LINUX AND ARCH_AMD64) +if (OS_LINUX) set (LINK_CONFIG_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") endif () From 4e9d37e033ca0964b64b89aa8d4607685fcaaf3a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 04:46:13 +0300 Subject: [PATCH 349/390] More convenient --- benchmark/hardware.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 693a69deb1f..bd932a57cda 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -36,14 +36,6 @@ if [[ ! -f clickhouse ]]; then $FASTER_DOWNLOAD "$AMD64_BIN_URL" elif [[ $CPU == aarch64 ]]; then $FASTER_DOWNLOAD "$AARCH64_BIN_URL" - - # Download configs. ARM version has no embedded configs. - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/users.xml - mkdir config.d - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/path.xml -O config.d/path.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/access_control.xml -O config.d/access_control.xml - wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/programs/server/config.d/log_to_console.xml -O config.d/log_to_console.xml else echo "Unsupported CPU type: $CPU" exit 1 @@ -60,7 +52,7 @@ if [[ ! -d data ]]; then if [[ ! -f $DATASET ]]; then $FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET" fi - + tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET fi From 23fd72a3deb31a44272386c4a4399a05c58151be Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 25 Sep 2020 04:59:01 +0300 Subject: [PATCH 350/390] Update adopters.md --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 6d57dfde9cd..0cbfdcb7d81 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -38,7 +38,7 @@ toc_title: Adopters | Deutsche Bank | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) | | Diva-e | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) | | Ecwid | E-commerce SaaS | Metrics, Logging | — | — | [Slides in Russian, April 2019](https://nastachku.ru/var/files/1/presentation/backend/2_Backend_6.pdf) | -| eBay | E-commerce | TBA | — | — | [Webinar, Sep 2020](https://altinity.com/webinarspage/2020/09/08/migrating-from-druid-to-next-gen-olap-on-clickhouse-ebays-experience) | +| eBay | E-commerce | Logs, Metrics and Events | — | — | [Official website, Sep 2020](https://tech.ebayinc.com/engineering/ou-online-analytical-processing/) | | Exness | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) | | FastNetMon | DDoS Protection | Main Product | | — | [Official website](https://fastnetmon.com/docs-fnm-advanced/fastnetmon-advanced-traffic-persistency/) | | Flipkart | e-Commerce | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=239) | From 1f6e55ff35615351318c661ef4c9a24c69498b7a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 05:03:58 +0300 Subject: [PATCH 351/390] Allow to run without /proc/self/maps --- programs/server/Server.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6d47e7e0364..b703d0ac6a7 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -339,16 +339,23 @@ int Server::main(const std::vector & /*args*/) { if (hasLinuxCapability(CAP_IPC_LOCK)) { - /// Get the memory area with (current) code segment. - /// It's better to lock only the code segment instead of calling "mlockall", - /// because otherwise debug info will be also locked in memory, and it can be huge. - auto [addr, len] = getMappedArea(reinterpret_cast(mainEntryClickHouseServer)); + try + { + /// Get the memory area with (current) code segment. + /// It's better to lock only the code segment instead of calling "mlockall", + /// because otherwise debug info will be also locked in memory, and it can be huge. + auto [addr, len] = getMappedArea(reinterpret_cast(mainEntryClickHouseServer)); - LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds."); - if (0 != mlock(addr, len)) - LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); - else - LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len)); + LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds."); + if (0 != mlock(addr, len)) + LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR)); + else + LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len)); + } + catch (...) + { + LOG_WARNING(log, "Cannot mlock: {}", getCurrentExceptionMessage(false)); + } } else { From 85bfdbad24760fb507dd6497c50a132073853af8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 05:12:51 +0300 Subject: [PATCH 352/390] Add a test for embedded configs --- ...erver_start_with_embedded_config.reference | 5 ++ ...house_server_start_with_embedded_config.sh | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.reference create mode 100755 tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh diff --git a/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.reference b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.reference new file mode 100644 index 00000000000..c3829d603de --- /dev/null +++ b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.reference @@ -0,0 +1,5 @@ +Starting clickhouse-server +Waiting for clickhouse-server to start +1 +Hello +World diff --git a/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh new file mode 100755 index 00000000000..68198ec6e16 --- /dev/null +++ b/tests/queries/0_stateless/01507_clickhouse_server_start_with_embedded_config.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +CLICKHOUSE_PORT_TCP=50111 +CLICKHOUSE_DATABASE=default + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +echo "Starting clickhouse-server" + +$PORT + +$CLICKHOUSE_BINARY server -- --tcp_port "$CLICKHOUSE_PORT_TCP" > server.log 2>&1 & +PID=$! + +function finish { + kill $PID + wait +} +trap finish EXIT + +echo "Waiting for clickhouse-server to start" + +for i in {1..30}; do + sleep 1 + $CLICKHOUSE_CLIENT --query "SELECT 1" 2>/dev/null && break + if [[ $i == 30 ]]; then + cat server.log + exit 1 + fi +done + +# Check access rights + +$CLICKHOUSE_CLIENT -n --query " + DROP DATABASE IF EXISTS test; + CREATE DATABASE test; + USE test; + + CREATE TABLE t (s String) ENGINE=TinyLog; + INSERT INTO t VALUES ('Hello'); + SELECT * FROM t; + DROP TABLE t; + + CREATE TEMPORARY TABLE t (s String); + INSERT INTO t VALUES ('World'); + SELECT * FROM t; +"; From 161ee1df3ed262191f6c10eff9046f4ab62db8e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 25 Sep 2020 05:27:13 +0300 Subject: [PATCH 353/390] Fail more quickly in clickhouse-test if server does not respond --- tests/clickhouse-test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index a3bed189d55..3f0c4bd155a 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -303,6 +303,14 @@ def run_tests_array(all_tests_with_params): clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) clickhouse_proc.communicate("SELECT 'Running test {suite}/{case} from pid={pid}';".format(pid = os.getpid(), case = case, suite = suite)) + if clickhouse_proc.returncode != 0: + failures += 1 + print(MSG_FAIL, end='') + print_test_time(0) + print(" - server does not respond to health check") + SERVER_DIED = True + break + reference_file = os.path.join(suite_dir, name) + '.reference' stdout_file = os.path.join(suite_tmp_dir, name) + '.stdout' stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr' From 23f06de0bef472ccf18b950c2afc7629c7266a65 Mon Sep 17 00:00:00 2001 From: John <108540613@qq.com> Date: Fri, 25 Sep 2020 11:48:25 +0800 Subject: [PATCH 354/390] Update in.md --- docs/zh/sql-reference/operators/in.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/sql-reference/operators/in.md b/docs/zh/sql-reference/operators/in.md index eaaa477fbe1..bcd3ca1fa18 100644 --- a/docs/zh/sql-reference/operators/in.md +++ b/docs/zh/sql-reference/operators/in.md @@ -3,7 +3,7 @@ machine_translated: true machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 --- -# 在运营商 {#select-in-operators} +# IN 操作符 {#select-in-operators} 该 `IN`, `NOT IN`, `GLOBAL IN`，和 `GLOBAL NOT IN` 运算符是单独复盖的，因为它们的功能相当丰富。 From 049f21091e35f5dbff1182128bcafe154700e120 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 25 Sep 2020 08:05:50 +0300 Subject: [PATCH 355/390] Update clickhouse-test --- tests/clickhouse-test | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3f0c4bd155a..36467f18d55 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -305,9 +305,7 @@ def run_tests_array(all_tests_with_params): if clickhouse_proc.returncode != 0: failures += 1 - print(MSG_FAIL, end='') - print_test_time(0) - print(" - server does not respond to health check") + print("Server does not respond to health check") SERVER_DIED = True break From 6d865449117afead01e3bce527577487f943d5d9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 10:55:41 +0300 Subject: [PATCH 356/390] Trying to supress race in LazyPipeFDs --- tests/tsan_suppressions.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/tsan_suppressions.txt b/tests/tsan_suppressions.txt index 912e0361bff..b12001f4f42 100644 --- a/tests/tsan_suppressions.txt +++ b/tests/tsan_suppressions.txt @@ -1 +1,4 @@ # Fortunately, we have no suppressions! + +# looks like a bug in clang-11 thread sanitizer, detects normal data race with random FD in this method +# race:DB::LazyPipeFDs::close From 61ab1a264dedcb84654efb9f2f69f977d712c1b8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 10:56:30 +0300 Subject: [PATCH 357/390] Suppress race --- tests/tsan_suppressions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsan_suppressions.txt b/tests/tsan_suppressions.txt index b12001f4f42..35c4ba856e6 100644 --- a/tests/tsan_suppressions.txt +++ b/tests/tsan_suppressions.txt @@ -1,4 +1,4 @@ # Fortunately, we have no suppressions! # looks like a bug in clang-11 thread sanitizer, detects normal data race with random FD in this method -# race:DB::LazyPipeFDs::close +race:DB::LazyPipeFDs::close From 4666ae3211667431d73d0b474693716bec3b12dd Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Fri, 25 Sep 2020 17:59:19 +0800 Subject: [PATCH 358/390] fix build bug for merge_selector2.cpp --- src/Storages/MergeTree/MergeSelector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeSelector.h b/src/Storages/MergeTree/MergeSelector.h index e460b8ae06a..fcdfcf5b890 100644 --- a/src/Storages/MergeTree/MergeSelector.h +++ b/src/Storages/MergeTree/MergeSelector.h @@ -44,7 +44,7 @@ public: /// Information about different TTLs for part. Can be used by /// TTLSelector to assign merges with TTL. - const MergeTreeDataPartTTLInfos * ttl_infos; + const MergeTreeDataPartTTLInfos * ttl_infos = nullptr; /// Part compression codec definition. ASTPtr compression_codec_desc; From 3f717ea1c471f9c15e617003c9a52ae6776552ba Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 25 Sep 2020 13:39:59 +0300 Subject: [PATCH 359/390] fix skip lists --- tests/queries/0_stateless/arcadia_skip_list.txt | 4 ++++ tests/queries/skip_list.json | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6d1c6444d1b..3191eb87fed 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -145,4 +145,8 @@ 01461_query_start_time_microseconds 01455_shard_leaf_max_rows_bytes_to_read 01505_distributed_local_type_conversion_enum +00604_show_create_database +00609_mv_index_in_in +00510_materizlized_view_and_deduplication_zookeeper +00738_lock_for_inner_table 01505_log_distributed_deadlock diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index abd6ed83304..26e5bbf78cf 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -89,10 +89,9 @@ "release-build": [ ], "database-ordinary": [ - "00604_show_create_database" - ], - "polymorphic-parts": [ - "00933_test_fix_extra_seek_on_compressed_cache", - "00446_clear_column_in_partition_zookeeper" + "00604_show_create_database", + "00609_mv_index_in_in", + "00510_materizlized_view_and_deduplication_zookeeper", + "00738_lock_for_inner_table" ] } From d3f3867445ed9aaf459b9931176f8e311baeacf4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 13:46:38 +0300 Subject: [PATCH 360/390] Update tsan_suppressions.txt --- tests/tsan_suppressions.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/tsan_suppressions.txt b/tests/tsan_suppressions.txt index 35c4ba856e6..668710a33d7 100644 --- a/tests/tsan_suppressions.txt +++ b/tests/tsan_suppressions.txt @@ -1,4 +1,2 @@ -# Fortunately, we have no suppressions! - # looks like a bug in clang-11 thread sanitizer, detects normal data race with random FD in this method race:DB::LazyPipeFDs::close From 97671a17719eb9c101095e01cac7baa282908126 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 14:27:00 +0300 Subject: [PATCH 361/390] Ugly fix for default database race --- programs/server/Server.cpp | 13 +++++++++---- src/Interpreters/Context.cpp | 12 ++++++++++++ src/Interpreters/Context.h | 3 +++ src/Interpreters/SystemLog.h | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b703d0ac6a7..6341653ee2f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -671,6 +671,10 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + /// Set current database name before loading tables and databases because + /// system logs may copy global context. + global_context->setCurrentDatabaseNameInGlobalContext(default_database); + LOG_INFO(log, "Loading metadata from {}", path); try @@ -678,11 +682,14 @@ int Server::main(const std::vector & /*args*/) loadMetadataSystem(*global_context); /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); + auto & database_catalog = DatabaseCatalog::instance(); /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) - attachSystemTablesServer(*DatabaseCatalog::instance().getSystemDatabase(), has_zookeeper); + attachSystemTablesServer(*database_catalog.getSystemDatabase(), has_zookeeper); /// Then, load remaining databases loadMetadata(*global_context, default_database); - DatabaseCatalog::instance().loadDatabases(); + database_catalog.loadDatabases(); + /// After loading validate that default database exists + database_catalog.assertDatabaseExists(default_database); } catch (...) { @@ -745,8 +752,6 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); - global_context->setCurrentDatabase(default_database); - if (has_zookeeper && config().has("distributed_ddl")) { /// DDL worker should be started after all tables were loaded diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 704b21f3a4a..be35c8a9184 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1088,6 +1088,18 @@ String Context::getInitialQueryId() const } +void Context::setCurrentDatabaseNameInGlobalContext(const String & name) +{ + if (global_context != this) + throw Exception("Cannot set current database for non global context, this method should be used during server initialization", ErrorCodes::LOGICAL_ERROR); + auto lock = getLock(); + + if (!current_database.empty()) + throw Exception("Default database name cannot be changed in global context without server restart", ErrorCodes::LOGICAL_ERROR); + + current_database = name; +} + void Context::setCurrentDatabase(const String & name) { DatabaseCatalog::instance().assertDatabaseExists(name); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 3d66ef239e7..bd5e17fe2e4 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -359,6 +359,9 @@ public: String getInitialQueryId() const; void setCurrentDatabase(const String & name); + /// Set current_database for global context. We don't validate that database + /// exists because it should be set before databases loading. + void setCurrentDatabaseNameInGlobalContext(const String & name); void setCurrentQueryId(const String & query_id); void killCurrentQuery(); diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 03b1b735cbc..2a0ce9cef53 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -438,7 +438,7 @@ void SystemLog::flushImpl(const std::vector & to_flush, ASTPtr query_ptr(insert.release()); // we need query context to do inserts to target table with MV containing subqueries or joins - auto insert_context = Context(context); + Context insert_context(context); insert_context.makeQueryContext(); InterpreterInsertQuery interpreter(query_ptr, insert_context); From b30e86016635da8bc0197ee3b93694c431ba33e5 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 25 Sep 2020 14:51:00 +0300 Subject: [PATCH 362/390] Update ASTColumnsTransformers.h --- src/Parsers/ASTColumnsTransformers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index ddf0d70dc35..4b7a933647e 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -53,7 +53,7 @@ public: ASTPtr clone() const override { auto replacement = std::make_shared(*this); - replacement->name = name; + replacement->children.clear(); replacement->expr = expr->clone(); replacement->children.push_back(replacement->expr); return replacement; From 8f21c73b763ce86d2032648d00325baf9945d116 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 25 Sep 2020 14:52:55 +0300 Subject: [PATCH 363/390] Revert "Update ASTColumnsTransformers.h" This reverts commit b30e86016635da8bc0197ee3b93694c431ba33e5. --- src/Parsers/ASTColumnsTransformers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index 4b7a933647e..ddf0d70dc35 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -53,7 +53,7 @@ public: ASTPtr clone() const override { auto replacement = std::make_shared(*this); - replacement->children.clear(); + replacement->name = name; replacement->expr = expr->clone(); replacement->children.push_back(replacement->expr); return replacement; From d4c8ba84102c948f1da37c3f00abd1ca6fa2a15f Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 25 Sep 2020 14:58:04 +0300 Subject: [PATCH 364/390] Update ASTColumnsTransformers.h --- src/Parsers/ASTColumnsTransformers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index ddf0d70dc35..4b7a933647e 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -53,7 +53,7 @@ public: ASTPtr clone() const override { auto replacement = std::make_shared(*this); - replacement->name = name; + replacement->children.clear(); replacement->expr = expr->clone(); replacement->children.push_back(replacement->expr); return replacement; From f5c38b34ae5f439a46c661413a429efc30aca05f Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 15:07:38 +0300 Subject: [PATCH 365/390] Fix race condition during write ahead log cleanup and rename --- src/Storages/MergeTree/MergeTreeData.h | 2 ++ src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp | 4 +++- src/Storages/StorageMergeTree.cpp | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 0fc5ec43048..1125eb32b66 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -717,6 +717,8 @@ protected: bool require_part_metadata; + /// Relative path data, changes during rename for ordinary databases use + /// under lockForShare if rename is possible. String relative_data_path; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index a5216e6fda3..11f23a5c110 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -56,10 +56,12 @@ void ReplicatedMergeTreeCleanupThread::run() void ReplicatedMergeTreeCleanupThread::iterate() { storage.clearOldPartsAndRemoveFromZK(); - storage.clearOldWriteAheadLogs(); { auto lock = storage.lockForShare(RWLockImpl::NO_QUERY, storage.getSettings()->lock_acquire_timeout_for_background_operations); + /// Both use relative_data_path which changes during rename, so we + /// do it under share lock + storage.clearOldWriteAheadLogs(); storage.clearOldTemporaryDirectories(); } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 347474753dc..55fb42b550e 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -919,11 +919,13 @@ BackgroundProcessingPoolTaskResult StorageMergeTree::mergeMutateTask() { { auto share_lock = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); + /// All use relative_data_path which changes during rename + /// so execute under share lock. clearOldPartsFromFilesystem(); clearOldTemporaryDirectories(); + clearOldWriteAheadLogs(); } clearOldMutations(); - clearOldWriteAheadLogs(); } ///TODO: read deduplicate option from table config From 86322fa3e1b86a2e96452805c18e440c29e6dab0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 15:36:02 +0300 Subject: [PATCH 366/390] Add test --- ...condition_rename_clear_zookeeper.reference | 1 + ...8_race_condition_rename_clear_zookeeper.sh | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference create mode 100755 tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference new file mode 100644 index 00000000000..13de30f45d1 --- /dev/null +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference @@ -0,0 +1 @@ +3000 diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh new file mode 100755 index 00000000000..2af1cb214a4 --- /dev/null +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames50" + + +$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_01508/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1" + + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(1000)" + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(1000, 1000)" + +$CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(2000, 1000)" + +for i in $(seq 1 50); do + prev_i=$((i - 1)) + $CLICKHOUSE_CLIENT --query "RENAME TABLE table_for_renames$prev_i TO table_for_renames$i" +done + +$CLICKHOUSE_CLIENT --query "SELECT COUNT() from table_for_renames50" + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames50" From 39c356f23061d08e9f662fd254c01a385e6facef Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:12:52 +0300 Subject: [PATCH 367/390] Change phone number --- docs/en/introduction/info.md | 10 ++++++++++ docs/ru/introduction/info.md | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docs/en/introduction/info.md diff --git a/docs/en/introduction/info.md b/docs/en/introduction/info.md new file mode 100644 index 00000000000..a397c40950d --- /dev/null +++ b/docs/en/introduction/info.md @@ -0,0 +1,10 @@ +--- +toc_priority: 100 +--- + +# Information support {#information-support} + +- Email address: +- Phone: +7-495-780-6510 + +[Original article](https://clickhouse.tech/docs/en/introduction/info/) \ No newline at end of file diff --git a/docs/ru/introduction/info.md b/docs/ru/introduction/info.md index 14e517eebae..a9398b8c9cd 100644 --- a/docs/ru/introduction/info.md +++ b/docs/ru/introduction/info.md @@ -7,6 +7,6 @@ toc_priority: 100 Информационная поддержка ClickHouse осуществляется на всей территории Российской Федерации без ограничений посредством использования телефонной связи и средств электронной почты на русском языке в круглосуточном режиме: - Адрес электронной почты: -- Телефон: 8-800-250-96-39 (звонки бесплатны из всех регионов России) +- Телефон: +7-495-780-6510 [Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/info/) From 54fa67838e95364c5169ccb67b8e7dfc0b90e337 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 16:32:00 +0300 Subject: [PATCH 368/390] Add flaky test --- tests/integration/test_flaky_check/__init__.py | 0 tests/integration/test_flaky_check/test.py | 5 +++++ 2 files changed, 5 insertions(+) create mode 100644 tests/integration/test_flaky_check/__init__.py create mode 100644 tests/integration/test_flaky_check/test.py diff --git a/tests/integration/test_flaky_check/__init__.py b/tests/integration/test_flaky_check/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_flaky_check/test.py b/tests/integration/test_flaky_check/test.py new file mode 100644 index 00000000000..4da85151c46 --- /dev/null +++ b/tests/integration/test_flaky_check/test.py @@ -0,0 +1,5 @@ +import random + + +def test_flaky(): + assert random.randint(0, 2) >= 1 From ada193e7b716f1ebd04abefe8fbbdcb225e1f766 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 16:33:33 +0300 Subject: [PATCH 369/390] Add diff --- tests/integration/test_ttl_move/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 377ee0e5d75..a4f8606011d 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -8,6 +8,7 @@ import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster +# test cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', From 6914a8667311b8f784c36b9a77c15580b469844c Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Fri, 25 Sep 2020 10:31:06 +0300 Subject: [PATCH 370/390] Put createBlockForSet declaration to header file. --- src/Interpreters/ActionsVisitor.cpp | 15 ++------------- src/Interpreters/ActionsVisitor.h | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index be040ff2c34..1d524669fd9 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -235,11 +235,7 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, co return header.cloneWithColumns(std::move(columns)); } -/** Create a block for set from literal. - * 'set_element_types' - types of what are on the left hand side of IN. - * 'right_arg' - Literal - Tuple or Array. - */ -static Block createBlockForSet( +Block createBlockForSet( const DataTypePtr & left_arg_type, const ASTPtr & right_arg, const DataTypes & set_element_types, @@ -280,14 +276,7 @@ static Block createBlockForSet( return block; } -/** Create a block for set from expression. - * 'set_element_types' - types of what are on the left hand side of IN. - * 'right_arg' - list of values: 1, 2, 3 or list of tuples: (1, 2), (3, 4), (5, 6). - * - * We need special implementation for ASTFunction, because in case, when we interpret - * large tuple or array as function, `evaluateConstantExpression` works extremely slow. - */ -static Block createBlockForSet( +Block createBlockForSet( const DataTypePtr & left_arg_type, const std::shared_ptr & right_arg, const DataTypes & set_element_types, diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index d8d85f1c0bf..98ea3f79fff 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -16,11 +16,37 @@ struct ExpressionAction; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; - /// The case of an explicit enumeration of values. +/// The case of an explicit enumeration of values. SetPtr makeExplicitSet( const ASTFunction * node, const Block & sample_block, bool create_ordered_set, const Context & context, const SizeLimits & limits, PreparedSets & prepared_sets); +/** Create a block for set from expression. + * 'set_element_types' - types of what are on the left hand side of IN. + * 'right_arg' - list of values: 1, 2, 3 or list of tuples: (1, 2), (3, 4), (5, 6). + * + * We need special implementation for ASTFunction, because in case, when we interpret + * large tuple or array as function, `evaluateConstantExpression` works extremely slow. + * + * Note: this and following functions are used in third-party applications in Arcadia, so + * they should be declared in header file. + * + */ +Block createBlockForSet( + const DataTypePtr & left_arg_type, + const std::shared_ptr & right_arg, + const DataTypes & set_element_types, + const Context & context); + +/** Create a block for set from literal. + * 'set_element_types' - types of what are on the left hand side of IN. + * 'right_arg' - Literal - Tuple or Array. + */ +Block createBlockForSet( + const DataTypePtr & left_arg_type, + const ASTPtr & right_arg, + const DataTypes & set_element_types, + const Context & context); /** For ActionsVisitor * A stack of ExpressionActions corresponding to nested lambda expressions. From af0bb5bd263d0b12600f0ea2d86b0cf6a6d76bf1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 19:15:34 +0300 Subject: [PATCH 371/390] One more time --- tests/integration/test_s3_with_proxy/test.py | 14 ++++++++++---- tests/integration/test_ttl_move/test.py | 1 - 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 9df209826f9..3d118266455 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -1,5 +1,6 @@ import logging import os +import time import pytest from helpers.cluster import ClickHouseCluster @@ -37,10 +38,15 @@ def cluster(): def check_proxy_logs(cluster, proxy_instance, http_methods={"POST", "PUT", "GET", "DELETE"}): - logs = cluster.get_container_logs(proxy_instance) - # Check that all possible interactions with Minio are present - for http_method in http_methods: - assert logs.find(http_method + " http://minio1") >= 0 + for i in range(10): + logs = cluster.get_container_logs(proxy_instance) + # Check that all possible interactions with Minio are present + for http_method in http_methods: + if logs.find(http_method + " http://minio1") >= 0: + return + time.sleep(1) + else: + assert False, "http method not found in logs" @pytest.mark.parametrize( diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index a4f8606011d..377ee0e5d75 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -8,7 +8,6 @@ import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -# test cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', From 8f9ec303154fc3e04222a4c999ec6f8e8a1bcf01 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 19:23:57 +0300 Subject: [PATCH 372/390] Delete flaky test --- tests/integration/test_flaky_check/__init__.py | 0 tests/integration/test_flaky_check/test.py | 5 ----- 2 files changed, 5 deletions(-) delete mode 100644 tests/integration/test_flaky_check/__init__.py delete mode 100644 tests/integration/test_flaky_check/test.py diff --git a/tests/integration/test_flaky_check/__init__.py b/tests/integration/test_flaky_check/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_flaky_check/test.py b/tests/integration/test_flaky_check/test.py deleted file mode 100644 index 4da85151c46..00000000000 --- a/tests/integration/test_flaky_check/test.py +++ /dev/null @@ -1,5 +0,0 @@ -import random - - -def test_flaky(): - assert random.randint(0, 2) >= 1 From 5d8de1f65d8b3728847ae6a5ae5aca8f6f070c2e Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 25 Sep 2020 20:31:01 +0300 Subject: [PATCH 373/390] Update ci_config.json --- tests/ci/ci_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 220d8d801ec..3d41e8495e9 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -237,7 +237,7 @@ "with_coverage": false } }, - "Functional stateful tests (release, DatabaseAtomic)": { + "Functional stateful tests (release, DatabaseOrdinary)": { "required_build_properties": { "compiler": "gcc-10", "package_type": "deb", @@ -345,7 +345,7 @@ "with_coverage": false } }, - "Functional stateless tests (release, DatabaseAtomic)": { + "Functional stateless tests (release, DatabaseOrdinary)": { "required_build_properties": { "compiler": "gcc-10", "package_type": "deb", From 35ab3657c3cd2e313279733f538f43fccbb903ac Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Sep 2020 21:39:55 +0300 Subject: [PATCH 374/390] Add flaky tests config --- tests/ci/ci_config.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 220d8d801ec..c01025ea865 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -441,6 +441,18 @@ "with_coverage": false } }, + "Integration tests flaky check (asan)": { + "required_build_properties": { + "compiler": "clang-11", + "package_type": "deb", + "build_type": "relwithdebuginfo", + "sanitizer": "address", + "bundled": "bundled", + "splitted": "unsplitted", + "clang-tidy": "disable", + "with_coverage": false + } + }, "Compatibility check": { "required_build_properties": { "compiler": "gcc-10", From 62c2c2000cd0c33d8a8b39d7c34a0104848af9b7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:00:54 +0300 Subject: [PATCH 375/390] Query obfuscator: development --- src/CMakeLists.txt | 2 +- src/Common/BitHelpers.h | 43 +- src/Common/StringUtils/StringUtils.h | 23 +- src/Parsers/obfuscateQueries.cpp | 934 ++++++++++++++++++ src/Parsers/obfuscateQueries.h | 29 + src/Parsers/tests/gtest_obfuscate_queries.cpp | 92 ++ 6 files changed, 1104 insertions(+), 19 deletions(-) create mode 100644 src/Parsers/obfuscateQueries.cpp create mode 100644 src/Parsers/obfuscateQueries.h create mode 100644 src/Parsers/tests/gtest_obfuscate_queries.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0016c51b7f8..40815228eac 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -415,6 +415,6 @@ if (ENABLE_TESTS AND USE_GTEST) -Wno-gnu-zero-variadic-macro-arguments ) - target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) + target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) add_check(unit_tests_dbms) endif () diff --git a/src/Common/BitHelpers.h b/src/Common/BitHelpers.h index 699e379b8d3..eac5fdac80e 100644 --- a/src/Common/BitHelpers.h +++ b/src/Common/BitHelpers.h @@ -1,22 +1,12 @@ #pragma once #include +#include #include #include #include -/** Returns log2 of number, rounded down. - * Compiles to single 'bsr' instruction on x86. - * For zero argument, result is unspecified. - */ -inline unsigned int bitScanReverse(unsigned int x) -{ - assert(x != 0); - return sizeof(unsigned int) * 8 - 1 - __builtin_clz(x); -} - - /** For zero argument, result is zero. * For arguments with most significand bit set, result is n. * For other arguments, returns value, rounded up to power of two. @@ -41,10 +31,9 @@ inline size_t roundUpToPowerOfTwoOrZero(size_t n) template -inline size_t getLeadingZeroBits(T x) +inline size_t getLeadingZeroBitsUnsafe(T x) { - if (!x) - return sizeof(x) * 8; + assert(x != 0); if constexpr (sizeof(T) <= sizeof(unsigned int)) { @@ -60,10 +49,32 @@ inline size_t getLeadingZeroBits(T x) } } + +template +inline size_t getLeadingZeroBits(T x) +{ + if (!x) + return sizeof(x) * 8; + + return getLeadingZeroBitsUnsafe(x); +} + +/** Returns log2 of number, rounded down. + * Compiles to single 'bsr' instruction on x86. + * For zero argument, result is unspecified. + */ +template +inline uint32_t bitScanReverse(T x) +{ + return sizeof(T) * 8 - 1 - getLeadingZeroBitsUnsafe(x); +} + // Unsafe since __builtin_ctz()-family explicitly state that result is undefined on x == 0 template inline size_t getTrailingZeroBitsUnsafe(T x) { + assert(x != 0); + if constexpr (sizeof(T) <= sizeof(unsigned int)) { return __builtin_ctz(x); @@ -88,8 +99,8 @@ inline size_t getTrailingZeroBits(T x) } /** Returns a mask that has '1' for `bits` LSB set: - * maskLowBits(3) => 00000111 - */ + * maskLowBits(3) => 00000111 + */ template inline T maskLowBits(unsigned char bits) { diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index a1e8fb79435..904e3035dd8 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -67,10 +67,19 @@ inline bool isASCII(char c) return static_cast(c) < 0x80; } +inline bool isLowerAlphaASCII(char c) +{ + return (c >= 'a' && c <= 'z'); +} + +inline bool isUpperAlphaASCII(char c) +{ + return (c >= 'A' && c <= 'Z'); +} + inline bool isAlphaASCII(char c) { - return (c >= 'a' && c <= 'z') - || (c >= 'A' && c <= 'Z'); + return isLowerAlphaASCII(c) || isUpperAlphaASCII(c); } inline bool isNumericASCII(char c) @@ -122,6 +131,16 @@ inline bool isPrintableASCII(char c) return uc >= 32 && uc <= 126; /// 127 is ASCII DEL. } +inline bool isPunctuationASCII(char c) +{ + uint8_t uc = c; + return (uc >= 33 && uc <= 47) + || (uc >= 58 && uc <= 64) + || (uc >= 91 && uc <= 96) + || (uc >= 123 && uc <= 125); +} + + inline bool isValidIdentifier(const std::string_view & str) { return !str.empty() && isValidIdentifierBegin(str[0]) && std::all_of(str.begin() + 1, str.end(), isWordCharASCII); diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp new file mode 100644 index 00000000000..66221005d77 --- /dev/null +++ b/src/Parsers/obfuscateQueries.cpp @@ -0,0 +1,934 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_MANY_TEMPORARY_COLUMNS; +} + + +namespace +{ + +const std::unordered_set keywords +{ + "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", + "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", + "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", + "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", + "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", + "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", + "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", + "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", + "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", + "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", + "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "FOR", "RANDOMIZED", + "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY" +}; + +const std::unordered_set keep_words +{ + "id", "name", "value", "num", + "Id", "Name", "Value", "Num", + "ID", "NAME", "VALUE", "NUM", +}; + +/// The list of nouns collected from here: http://www.desiquintans.com/nounlist, Public domain. +std::initializer_list nouns +{ +"aardvark", "abacus", "abbey", "abbreviation", "abdomen", "ability", "abnormality", "abolishment", "abortion", +"abrogation", "absence", "abundance", "abuse", "academics", "academy", "accelerant", "accelerator", "accent", "acceptance", "access", +"accessory", "accident", "accommodation", "accompanist", "accomplishment", "accord", "accordance", "accordion", "account", "accountability", +"accountant", "accounting", "accuracy", "accusation", "acetate", "achievement", "achiever", "acid", "acknowledgment", "acorn", "acoustics", +"acquaintance", "acquisition", "acre", "acrylic", "act", "action", "activation", "activist", "activity", "actor", "actress", "acupuncture", +"ad", "adaptation", "adapter", "addiction", "addition", "address", "adjective", "adjustment", "admin", "administration", "administrator", +"admire", "admission", "adobe", "adoption", "adrenalin", "adrenaline", "adult", "adulthood", "advance", "advancement", "advantage", "advent", +"adverb", "advertisement", "advertising", "advice", "adviser", "advocacy", "advocate", "affair", "affect", "affidavit", "affiliate", +"affinity", "afoul", "afterlife", "aftermath", "afternoon", "aftershave", "aftershock", "afterthought", "age", "agency", "agenda", "agent", +"aggradation", "aggression", "aglet", "agony", "agreement", "agriculture", "aid", "aide", "aim", "air", "airbag", "airbus", "aircraft", +"airfare", "airfield", "airforce", "airline", "airmail", "airman", "airplane", "airport", "airship", "airspace", "alarm", "alb", "albatross", +"album", "alcohol", "alcove", "alder", "ale", "alert", "alfalfa", "algebra", "algorithm", "alias", "alibi", "alien", "allegation", "allergist", +"alley", "alliance", "alligator", "allocation", "allowance", "alloy", "alluvium", "almanac", "almighty", "almond", "alpaca", "alpenglow", +"alpenhorn", "alpha", "alphabet", "altar", "alteration", "alternative", "altitude", "alto", "aluminium", "aluminum", "amazement", "amazon", +"ambassador", "amber", "ambience", "ambiguity", "ambition", "ambulance", "amendment", "amenity", "ammunition", "amnesty", "amount", "amusement", +"anagram", "analgesia", "analog", "analogue", "analogy", "analysis", "analyst", "analytics", "anarchist", "anarchy", "anatomy", "ancestor", +"anchovy", "android", "anesthesiologist", "anesthesiology", "angel", "anger", "angina", "angiosperm", "angle", "angora", "angstrom", +"anguish", "animal", "anime", "anise", "ankle", "anklet", "anniversary", "announcement", "annual", "anorak", "answer", "ant", "anteater", +"antecedent", "antechamber", "antelope", "antennae", "anterior", "anthropology", "antibody", "anticipation", "anticodon", "antigen", +"antique", "antiquity", "antler", "antling", "anxiety", "anybody", "anyone", "anything", "anywhere", "apartment", "ape", "aperitif", +"apology", "app", "apparatus", "apparel", "appeal", "appearance", "appellation", "appendix", "appetiser", "appetite", "appetizer", "applause", +"apple", "applewood", "appliance", "application", "appointment", "appreciation", "apprehension", "approach", "appropriation", "approval", +"apricot", "apron", "apse", "aquarium", "aquifer", "arcade", "arch", "archaeologist", "archaeology", "archeology", "archer", +"architect", "architecture", "archives", "area", "arena", "argument", "arithmetic", "ark", "arm", "armadillo", "armament", +"armchair", "armoire", "armor", "armour", "armpit", "armrest", "army", "arrangement", "array", "arrest", "arrival", "arrogance", "arrow", +"art", "artery", "arthur", "artichoke", "article", "artifact", "artificer", "artist", "ascend", "ascent", "ascot", "ash", "ashram", "ashtray", +"aside", "asparagus", "aspect", "asphalt", "aspic", "ass", "assassination", "assault", "assembly", "assertion", "assessment", "asset", +"assignment", "assist", "assistance", "assistant", "associate", "association", "assumption", "assurance", "asterisk", "astrakhan", "astrolabe", +"astrologer", "astrology", "astronomy", "asymmetry", "atelier", "atheist", "athlete", "athletics", "atmosphere", "atom", "atrium", "attachment", +"attack", "attacker", "attainment", "attempt", "attendance", "attendant", "attention", "attenuation", "attic", "attitude", "attorney", +"attraction", "attribute", "auction", "audience", "audit", "auditorium", "aunt", "authentication", "authenticity", "author", "authorisation", +"authority", "authorization", "auto", "autoimmunity", "automation", "automaton", "autumn", "availability", "avalanche", "avenue", "average", +"avocado", "award", "awareness", "awe", "axis", "azimuth", "babe", "baboon", "babushka", "baby", "bachelor", "back", "backbone", +"backburn", "backdrop", "background", "backpack", "backup", "backyard", "bacon", "bacterium", "badge", "badger", "bafflement", "bag", +"bagel", "baggage", "baggie", "baggy", "bagpipe", "bail", "bait", "bake", "baker", "bakery", "bakeware", "balaclava", "balalaika", "balance", +"balcony", "ball", "ballet", "balloon", "balloonist", "ballot", "ballpark", "bamboo", "ban", "banana", "band", "bandana", "bandanna", +"bandolier", "bandwidth", "bangle", "banjo", "bank", "bankbook", "banker", "banking", "bankruptcy", "banner", "banquette", "banyan", +"baobab", "bar", "barbecue", "barbeque", "barber", "barbiturate", "bargain", "barge", "baritone", "barium", "bark", "barley", "barn", +"barometer", "barracks", "barrage", "barrel", "barrier", "barstool", "bartender", "base", "baseball", "baseboard", "baseline", "basement", +"basics", "basil", "basin", "basis", "basket", "basketball", "bass", "bassinet", "bassoon", "bat", "bath", "bather", "bathhouse", "bathrobe", +"bathroom", "bathtub", "battalion", "batter", "battery", "batting", "battle", "battleship", "bay", "bayou", "beach", "bead", "beak", +"beam", "bean", "beancurd", "beanie", "beanstalk", "bear", "beard", "beast", "beastie", "beat", "beating", "beauty", "beaver", "beck", +"bed", "bedrock", "bedroom", "bee", "beech", "beef", "beer", "beet", "beetle", "beggar", "beginner", "beginning", "begonia", "behalf", +"behavior", "behaviour", "beheading", "behest", "behold", "being", "belfry", "belief", "believer", "bell", "belligerency", "bellows", +"belly", "belt", "bench", "bend", "beneficiary", "benefit", "beret", "berry", "bestseller", "bet", "beverage", "beyond", +"bias", "bibliography", "bicycle", "bid", "bidder", "bidding", "bidet", "bifocals", "bijou", "bike", "bikini", "bill", "billboard", "billing", +"billion", "bin", "binoculars", "biology", "biopsy", "biosphere", "biplane", "birch", "bird", "birdbath", "birdcage", +"birdhouse", "birth", "birthday", "biscuit", "bit", "bite", "bitten", "bitter", "black", "blackberry", "blackbird", "blackboard", "blackfish", +"blackness", "bladder", "blade", "blame", "blank", "blanket", "blast", "blazer", "blend", "blessing", "blight", "blind", "blinker", "blister", +"blizzard", "block", "blocker", "blog", "blogger", "blood", "bloodflow", "bloom", "bloomer", "blossom", "blouse", "blow", "blowgun", +"blowhole", "blue", "blueberry", "blush", "boar", "board", "boat", "boatload", "boatyard", "bob", "bobcat", "body", "bog", "bolero", +"bolt", "bomb", "bomber", "bombing", "bond", "bonding", "bondsman", "bone", "bonfire", "bongo", "bonnet", "bonsai", "bonus", "boogeyman", +"book", "bookcase", "bookend", "booking", "booklet", "bookmark", "boolean", "boom", "boon", "boost", "booster", "boot", "bootee", "bootie", +"booty", "border", "bore", "borrower", "borrowing", "bosom", "boss", "botany", "bother", "bottle", "bottling", "bottom", "bottom-line", +"boudoir", "bough", "boulder", "boulevard", "boundary", "bouquet", "bourgeoisie", "bout", "boutique", "bow", "bower", "bowl", "bowler", +"bowling", "bowtie", "box", "boxer", "boxspring", "boy", "boycott", "boyfriend", "boyhood", "boysenberry", "bra", "brace", "bracelet", +"bracket", "brain", "brake", "bran", "branch", "brand", "brandy", "brass", "brassiere", "bratwurst", "bread", "breadcrumb", "breadfruit", +"break", "breakdown", "breakfast", "breakpoint", "breakthrough", "breast", "breastplate", "breath", "breeze", "brewer", "bribery", "brick", +"bricklaying", "bride", "bridge", "brief", "briefing", "briefly", "briefs", "brilliant", "brink", "brisket", "broad", "broadcast", "broccoli", +"brochure", "brocolli", "broiler", "broker", "bronchitis", "bronco", "bronze", "brooch", "brood", "brook", "broom", "brother", "brother-in-law", +"brow", "brown", "brownie", "browser", "browsing", "brunch", "brush", "brushfire", "brushing", "bubble", "buck", "bucket", "buckle", +"buckwheat", "bud", "buddy", "budget", "buffalo", "buffer", "buffet", "bug", "buggy", "bugle", "builder", "building", "bulb", "bulk", +"bull", "bulldozer", "bullet", "bump", "bumper", "bun", "bunch", "bungalow", "bunghole", "bunkhouse", "burden", "bureau", +"burglar", "burial", "burlesque", "burn", "burning", "burrito", "burro", "burrow", "burst", "bus", "bush", "business", "businessman", +"bust", "bustle", "butane", "butcher", "butler", "butter", "butterfly", "button", "buy", "buyer", "buying", "buzz", "buzzard", "c-clamp", +"cabana", "cabbage", "cabin", "cabinet", "cable", "caboose", "cacao", "cactus", "caddy", "cadet", "cafe", "caffeine", "caftan", "cage", +"cake", "calcification", "calculation", "calculator", "calculus", "calendar", "calf", "caliber", "calibre", "calico", "call", "calm", +"calorie", "camel", "cameo", "camera", "camp", "campaign", "campaigning", "campanile", "camper", "campus", "can", "canal", "cancer", +"candelabra", "candidacy", "candidate", "candle", "candy", "cane", "cannibal", "cannon", "canoe", "canon", "canopy", "cantaloupe", "canteen", +"canvas", "cap", "capability", "capacity", "cape", "caper", "capital", "capitalism", "capitulation", "capon", "cappelletti", "cappuccino", +"captain", "caption", "captor", "car", "carabao", "caramel", "caravan", "carbohydrate", "carbon", "carboxyl", "card", "cardboard", "cardigan", +"care", "career", "cargo", "caribou", "carload", "carnation", "carnival", "carol", "carotene", "carp", "carpenter", "carpet", "carpeting", +"carport", "carriage", "carrier", "carrot", "carry", "cart", "cartel", "carter", "cartilage", "cartload", "cartoon", "cartridge", "carving", +"cascade", "case", "casement", "cash", "cashew", "cashier", "casino", "casket", "cassava", "casserole", "cassock", "cast", "castanet", +"castle", "casualty", "cat", "catacomb", "catalogue", "catalysis", "catalyst", "catamaran", "catastrophe", "catch", "catcher", "category", +"caterpillar", "cathedral", "cation", "catsup", "cattle", "cauliflower", "causal", "cause", "causeway", "caution", "cave", "caviar", +"cayenne", "ceiling", "celebration", "celebrity", "celeriac", "celery", "cell", "cellar", "cello", "celsius", "cement", "cemetery", "cenotaph", +"census", "cent", "center", "centimeter", "centre", "centurion", "century", "cephalopod", "ceramic", "ceramics", "cereal", "ceremony", +"certainty", "certificate", "certification", "cesspool", "chafe", "chain", "chainstay", "chair", "chairlift", "chairman", "chairperson", +"chaise", "chalet", "chalice", "chalk", "challenge", "chamber", "champagne", "champion", "championship", "chance", "chandelier", "change", +"channel", "chaos", "chap", "chapel", "chaplain", "chapter", "character", "characteristic", "characterization", "chard", "charge", "charger", +"charity", "charlatan", "charm", "charset", "chart", "charter", "chasm", "chassis", "chastity", "chasuble", "chateau", "chatter", "chauffeur", +"chauvinist", "check", "checkbook", "checking", "checkout", "checkroom", "cheddar", "cheek", "cheer", "cheese", "cheesecake", "cheetah", +"chef", "chem", "chemical", "chemistry", "chemotaxis", "cheque", "cherry", "chess", "chest", "chestnut", "chick", "chicken", "chicory", +"chief", "chiffonier", "child", "childbirth", "childhood", "chili", "chill", "chime", "chimpanzee", "chin", "chinchilla", "chino", "chip", +"chipmunk", "chivalry", "chive", "chives", "chocolate", "choice", "choir", "choker", "cholesterol", "choosing", "chop", +"chops", "chopstick", "chopsticks", "chord", "chorus", "chow", "chowder", "chrome", "chromolithograph", "chronicle", "chronograph", "chronometer", +"chrysalis", "chub", "chuck", "chug", "church", "churn", "chutney", "cicada", "cigarette", "cilantro", "cinder", "cinema", "cinnamon", +"circadian", "circle", "circuit", "circulation", "circumference", "circumstance", "cirrhosis", "cirrus", "citizen", "citizenship", "citron", +"citrus", "city", "civilian", "civilisation", "civilization", "claim", "clam", "clamp", "clan", "clank", "clapboard", "clarification", +"clarinet", "clarity", "clasp", "class", "classic", "classification", "classmate", "classroom", "clause", "clave", "clavicle", "clavier", +"claw", "clay", "cleaner", "clearance", "clearing", "cleat", "cleavage", "clef", "cleft", "clergyman", "cleric", "clerk", "click", "client", +"cliff", "climate", "climb", "clinic", "clip", "clipboard", "clipper", "cloak", "cloakroom", "clock", "clockwork", "clogs", "cloister", +"clone", "close", "closet", "closing", "closure", "cloth", "clothes", "clothing", "cloud", "cloudburst", "clove", "clover", "cloves", +"club", "clue", "cluster", "clutch", "coach", "coal", "coalition", "coast", "coaster", "coat", "cob", "cobbler", "cobweb", +"cock", "cockpit", "cockroach", "cocktail", "cocoa", "coconut", "cod", "code", "codepage", "codling", "codon", "codpiece", "coevolution", +"cofactor", "coffee", "coffin", "cohesion", "cohort", "coil", "coin", "coincidence", "coinsurance", "coke", "cold", "coleslaw", "coliseum", +"collaboration", "collagen", "collapse", "collar", "collard", "collateral", "colleague", "collection", "collectivisation", "collectivization", +"collector", "college", "collision", "colloquy", "colon", "colonial", "colonialism", "colonisation", "colonization", "colony", "color", +"colorlessness", "colt", "column", "columnist", "comb", "combat", "combination", "combine", "comeback", "comedy", "comestible", "comfort", +"comfortable", "comic", "comics", "comma", "command", "commander", "commandment", "comment", "commerce", "commercial", "commission", +"commitment", "committee", "commodity", "common", "commonsense", "commotion", "communicant", "communication", "communion", "communist", +"community", "commuter", "company", "comparison", "compass", "compassion", "compassionate", "compensation", "competence", "competition", +"competitor", "complaint", "complement", "completion", "complex", "complexity", "compliance", "complication", "complicity", "compliment", +"component", "comportment", "composer", "composite", "composition", "compost", "comprehension", "compress", "compromise", "comptroller", +"compulsion", "computer", "comradeship", "con", "concentrate", "concentration", "concept", "conception", "concern", "concert", "conclusion", +"concrete", "condition", "conditioner", "condominium", "condor", "conduct", "conductor", "cone", "confectionery", "conference", "confidence", +"confidentiality", "configuration", "confirmation", "conflict", "conformation", "confusion", "conga", "congo", "congregation", "congress", +"congressman", "congressperson", "conifer", "connection", "connotation", "conscience", "consciousness", "consensus", "consent", "consequence", +"conservation", "conservative", "consideration", "consignment", "consist", "consistency", "console", "consonant", "conspiracy", "conspirator", +"constant", "constellation", "constitution", "constraint", "construction", "consul", "consulate", "consulting", "consumer", "consumption", +"contact", "contact lens", "contagion", "container", "content", "contention", "contest", "context", "continent", "contingency", "continuity", +"contour", "contract", "contractor", "contrail", "contrary", "contrast", "contribution", "contributor", "control", "controller", "controversy", +"convection", "convenience", "convention", "conversation", "conversion", "convert", "convertible", "conviction", "cook", "cookbook", +"cookie", "cooking", "coonskin", "cooperation", "coordination", "coordinator", "cop", "cope", "copper", "copy", "copying", +"copyright", "copywriter", "coral", "cord", "corduroy", "core", "cork", "cormorant", "corn", "corner", "cornerstone", "cornet", "cornflakes", +"cornmeal", "corporal", "corporation", "corporatism", "corps", "corral", "correspondence", "correspondent", "corridor", "corruption", +"corsage", "cosset", "cost", "costume", "cot", "cottage", "cotton", "couch", "cougar", "cough", "council", "councilman", "councilor", +"councilperson", "counsel", "counseling", "counselling", "counsellor", "counselor", "count", "counter", "counterpart", +"counterterrorism", "countess", "country", "countryside", "county", "couple", "coupon", "courage", "course", "court", "courthouse", "courtroom", +"cousin", "covariate", "cover", "coverage", "coverall", "cow", "cowbell", "cowboy", "coyote", "crab", "crack", "cracker", "crackers", +"cradle", "craft", "craftsman", "cranberry", "crane", "cranky", "crap", "crash", "crate", "cravat", "craw", "crawdad", "crayfish", "crayon", +"crazy", "cream", "creation", "creationism", "creationist", "creative", "creativity", "creator", "creature", "creche", "credential", +"credenza", "credibility", "credit", "creditor", "creek", "creme brulee", "crepe", "crest", "crew", "crewman", "crewmate", "crewmember", +"crewmen", "cria", "crib", "cribbage", "cricket", "cricketer", "crime", "criminal", "crinoline", "crisis", "crisp", "criteria", "criterion", +"critic", "criticism", "crocodile", "crocus", "croissant", "crook", "crop", "cross", "crotch", +"croup", "crow", "crowd", "crown", "crucifixion", "crude", "cruelty", "cruise", "crumb", "crunch", "crusader", "crush", "crust", "cry", +"crystal", "crystallography", "cub", "cube", "cuckoo", "cucumber", "cue", "cuisine", "cultivar", "cultivator", "culture", +"culvert", "cummerbund", "cup", "cupboard", "cupcake", "cupola", "curd", "cure", "curio", "curiosity", "curl", "curler", "currant", "currency", +"current", "curriculum", "curry", "curse", "cursor", "curtailment", "curtain", "curve", "cushion", "custard", "custody", "custom", "customer", +"cut", "cuticle", "cutlet", "cutover", "cutting", "cyclamen", "cycle", "cyclone", "cyclooxygenase", "cygnet", "cylinder", "cymbal", "cynic", +"cyst", "cytokine", "cytoplasm", "dad", "daddy", "daffodil", "dagger", "dahlia", "daikon", "daily", "dairy", "daisy", "dam", "damage", +"dame", "damn", "dance", "dancer", "dancing", "dandelion", "danger", "dare", "dark", "darkness", "darn", "dart", "dash", "dashboard", +"data", "database", "date", "daughter", "dawn", "day", "daybed", "daylight", "dead", "deadline", "deal", "dealer", "dealing", "dearest", +"death", "deathwatch", "debate", "debris", "debt", "debtor", "decade", "decadence", "decency", "decimal", "decision", "decision-making", +"deck", "declaration", "declination", "decline", "decoder", "decongestant", "decoration", "decrease", "decryption", "dedication", "deduce", +"deduction", "deed", "deep", "deer", "default", "defeat", "defendant", "defender", "defense", "deficit", "definition", "deformation", +"degradation", "degree", "delay", "deliberation", "delight", "delivery", "demand", "democracy", "democrat", "demon", "demur", "den", +"denim", "denominator", "density", "dentist", "deodorant", "department", "departure", "dependency", "dependent", "deployment", "deposit", +"deposition", "depot", "depression", "depressive", "depth", "deputy", "derby", "derivation", "derivative", "derrick", "descendant", "descent", +"description", "desert", "design", "designation", "designer", "desire", "desk", "desktop", "dessert", "destination", "destiny", "destroyer", +"destruction", "detail", "detainee", "detainment", "detection", "detective", "detector", "detention", "determination", "detour", "devastation", +"developer", "developing", "development", "developmental", "deviance", "deviation", "device", "devil", "dew", "dhow", "diabetes", "diadem", +"diagnosis", "diagram", "dial", "dialect", "dialogue", "diam", "diamond", "diaper", "diaphragm", "diarist", "diary", "dibble", "dick", +"dickey", "dictaphone", "dictator", "diction", "dictionary", "die", "diesel", "diet", "difference", "differential", "difficulty", "diffuse", +"dig", "digestion", "digestive", "digger", "digging", "digit", "dignity", "dilapidation", "dill", "dilution", "dime", "dimension", "dimple", +"diner", "dinghy", "dining", "dinner", "dinosaur", "dioxide", "dip", "diploma", "diplomacy", "dipstick", "direction", "directive", "director", +"directory", "dirndl", "dirt", "disability", "disadvantage", "disagreement", "disappointment", "disarmament", "disaster", "discharge", +"discipline", "disclaimer", "disclosure", "disco", "disconnection", "discount", "discourse", "discovery", "discrepancy", "discretion", +"discrimination", "discussion", "disdain", "disease", "disembodiment", "disengagement", "disguise", "disgust", "dish", "dishwasher", +"disk", "disparity", "dispatch", "displacement", "display", "disposal", "disposer", "disposition", "dispute", "disregard", "disruption", +"dissemination", "dissonance", "distance", "distinction", "distortion", "distribution", "distributor", "district", "divalent", "divan", +"diver", "diversity", "divide", "dividend", "divider", "divine", "diving", "division", "divorce", "doc", "dock", "doctor", "doctorate", +"doctrine", "document", "documentary", "documentation", "doe", "dog", "doggie", "dogsled", "dogwood", "doing", "doll", "dollar", "dollop", +"dolman", "dolor", "dolphin", "domain", "dome", "domination", "donation", "donkey", "donor", "donut", "door", "doorbell", "doorknob", +"doorpost", "doorway", "dory", "dose", "dot", "double", "doubling", "doubt", "doubter", "dough", "doughnut", "down", "downfall", "downforce", +"downgrade", "download", "downstairs", "downtown", "downturn", "dozen", "draft", "drag", "dragon", "dragonfly", "dragonfruit", "dragster", +"drain", "drainage", "drake", "drama", "dramaturge", "drapes", "draw", "drawbridge", "drawer", "drawing", "dream", "dreamer", "dredger", +"dress", "dresser", "dressing", "drill", "drink", "drinking", "drive", "driver", "driveway", "driving", "drizzle", "dromedary", "drop", +"drudgery", "drug", "drum", "drummer", "drunk", "dryer", "duck", "duckling", "dud", "dude", "due", "duel", "dueling", "duffel", "dugout", +"dulcimer", "dumbwaiter", "dump", "dump truck", "dune", "dune buggy", "dungarees", "dungeon", "duplexer", "duration", "durian", "dusk", +"dust", "dust storm", "duster", "duty", "dwarf", "dwell", "dwelling", "dynamics", "dynamite", "dynamo", "dynasty", "dysfunction", "e-book", +"eagle", "eaglet", "ear", "eardrum", "earmuffs", "earnings", "earplug", "earring", "earrings", "earth", "earthquake", +"earthworm", "ease", "easel", "east", "eating", "eaves", "eavesdropper", "ecclesia", "echidna", "eclipse", "ecliptic", "ecology", "economics", +"economy", "ecosystem", "ectoderm", "ectodermal", "ecumenist", "eddy", "edge", "edger", "edible", "editing", "edition", "editor", "editorial", +"education", "eel", "effacement", "effect", "effective", "effectiveness", "effector", "efficacy", "efficiency", "effort", "egg", "egghead", +"eggnog", "eggplant", "ego", "eicosanoid", "ejector", "elbow", "elderberry", "election", "electricity", "electrocardiogram", "electronics", +"element", "elephant", "elevation", "elevator", "eleventh", "elf", "elicit", "eligibility", "elimination", "elite", "elixir", "elk", +"ellipse", "elm", "elongation", "elver", "email", "emanate", "embarrassment", "embassy", "embellishment", "embossing", "embryo", "emerald", +"emergence", "emergency", "emergent", "emery", "emission", "emitter", "emotion", "emphasis", "empire", "employ", "employee", "employer", +"employment", "empowerment", "emu", "enactment", "encirclement", "enclave", "enclosure", "encounter", "encouragement", "encyclopedia", +"end", "endive", "endoderm", "endorsement", "endothelium", "endpoint", "enemy", "energy", "enforcement", "engagement", "engine", "engineer", +"engineering", "enigma", "enjoyment", "enquiry", "enrollment", "enterprise", "entertainment", "enthusiasm", "entirety", "entity", "entrance", +"entree", "entrepreneur", "entry", "envelope", "environment", "envy", "enzyme", "epauliere", "epee", "ephemera", "ephemeris", "ephyra", +"epic", "episode", "epithelium", "epoch", "eponym", "epoxy", "equal", "equality", "equation", "equinox", "equipment", "equity", "equivalent", +"era", "eraser", "erection", "erosion", "error", "escalator", "escape", "escort", "espadrille", "espalier", "essay", "essence", "essential", +"establishment", "estate", "estimate", "estrogen", "estuary", "eternity", "ethernet", "ethics", "ethnicity", "ethyl", "euphonium", "eurocentrism", +"evaluation", "evaluator", "evaporation", "eve", "evening", "event", "everybody", "everyone", "everything", "eviction", +"evidence", "evil", "evocation", "evolution", "exaggeration", "exam", "examination", "examiner", "example", +"exasperation", "excellence", "exception", "excerpt", "excess", "exchange", "excitement", "exclamation", "excursion", "excuse", "execution", +"executive", "executor", "exercise", "exhaust", "exhaustion", "exhibit", "exhibition", "exile", "existence", "exit", "exocrine", "expansion", +"expansionism", "expectancy", "expectation", "expedition", "expense", "experience", "experiment", "experimentation", "expert", "expertise", +"explanation", "exploration", "explorer", "explosion", "export", "expose", "exposition", "exposure", "expression", "extension", "extent", +"exterior", "external", "extinction", "extreme", "extremist", "eye", "eyeball", "eyebrow", "eyebrows", "eyeglasses", "eyelash", "eyelashes", +"eyelid", "eyelids", "eyeliner", "eyestrain", "eyrie", "fabric", "face", "facelift", "facet", "facility", "facsimile", "fact", "factor", +"factory", "faculty", "fahrenheit", "fail", "failure", "fairness", "fairy", "faith", "faithful", "fall", "fallacy", "fame", +"familiar", "familiarity", "family", "fan", "fang", "fanlight", "fanny", "fantasy", "farm", "farmer", "farming", "farmland", +"farrow", "fascia", "fashion", "fat", "fate", "father", "fatigue", "fatigues", "faucet", "fault", "fav", "fava", "favor", +"favorite", "fawn", "fax", "fear", "feast", "feather", "feature", "fedelini", "federation", "fedora", "fee", "feed", "feedback", "feeding", +"feel", "feeling", "fellow", "felony", "female", "fen", "fence", "fencing", "fender", "feng", "fennel", "ferret", "ferry", "ferryboat", +"fertilizer", "festival", "fetus", "few", "fiber", "fiberglass", "fibre", "fibroblast", "fibrosis", "ficlet", "fiction", "fiddle", "field", +"fiery", "fiesta", "fifth", "fig", "fight", "fighter", "figure", "figurine", "file", "filing", "fill", "fillet", "filly", "film", "filter", +"filth", "final", "finance", "financing", "finding", "fine", "finer", "finger", "fingerling", "fingernail", "finish", "finisher", "fir", +"fire", "fireman", "fireplace", "firewall", "firm", "first", "fish", "fishbone", "fisherman", "fishery", "fishing", "fishmonger", "fishnet", +"fisting", "fit", "fitness", "fix", "fixture", "flag", "flair", "flame", "flan", "flanker", "flare", "flash", "flat", "flatboat", "flavor", +"flax", "fleck", "fledgling", "fleece", "flesh", "flexibility", "flick", "flicker", "flight", "flint", "flintlock", "flock", +"flood", "floodplain", "floor", "floozie", "flour", "flow", "flower", "flu", "flugelhorn", "fluke", "flume", "flung", "flute", "fly", +"flytrap", "foal", "foam", "fob", "focus", "fog", "fold", "folder", "folk", "folklore", "follower", "following", "fondue", "font", "food", +"foodstuffs", "fool", "foot", "footage", "football", "footnote", "footprint", "footrest", "footstep", "footstool", "footwear", "forage", +"forager", "foray", "force", "ford", "forearm", "forebear", "forecast", "forehead", "foreigner", "forelimb", "forest", "forestry", "forever", +"forgery", "fork", "form", "formal", "formamide", "format", "formation", "former", "formicarium", "formula", "fort", "forte", "fortnight", +"fortress", "fortune", "forum", "foundation", "founder", "founding", "fountain", "fourths", "fowl", "fox", "foxglove", "fraction", "fragrance", +"frame", "framework", "fratricide", "fraud", "fraudster", "freak", "freckle", "freedom", "freelance", "freezer", "freezing", "freight", +"freighter", "frenzy", "freon", "frequency", "fresco", "friction", "fridge", "friend", "friendship", "fries", "frigate", "fright", "fringe", +"fritter", "frock", "frog", "front", "frontier", "frost", "frosting", "frown", "fruit", "frustration", "fry", "fuck", "fuel", "fugato", +"fulfillment", "full", "fun", "function", "functionality", "fund", "funding", "fundraising", "funeral", "fur", "furnace", "furniture", +"furry", "fusarium", "futon", "future", "gadget", "gaffe", "gaffer", "gain", "gaiters", "gale", "gallery", "galley", +"gallon", "galoshes", "gambling", "game", "gamebird", "gaming", "gander", "gang", "gap", "garage", "garb", "garbage", "garden", +"garlic", "garment", "garter", "gas", "gasket", "gasoline", "gasp", "gastronomy", "gastropod", "gate", "gateway", "gather", "gathering", +"gator", "gauge", "gauntlet", "gavel", "gazebo", "gazelle", "gear", "gearshift", "geek", "gel", "gelatin", "gelding", "gem", "gemsbok", +"gender", "gene", "general", "generation", "generator", "generosity", "genetics", "genie", "genius", "genocide", "genre", "gentleman", +"geography", "geology", "geometry", "geranium", "gerbil", "gesture", "geyser", "gherkin", "ghost", "giant", "gift", "gig", "gigantism", +"giggle", "ginger", "gingerbread", "ginseng", "giraffe", "girdle", "girl", "girlfriend", "git", "glacier", "gladiolus", "glance", "gland", +"glass", "glasses", "glee", "glen", "glider", "gliding", "glimpse", "globe", "glockenspiel", "gloom", "glory", "glove", "glow", "glucose", +"glue", "glut", "glutamate", "gnat", "gnu", "goal", "goat", "gobbler", "god", "goddess", "godfather", "godmother", "godparent", +"goggles", "going", "gold", "goldfish", "golf", "gondola", "gong", "good", "goodbye", "goodie", "goodness", "goodnight", +"goodwill", "goose", "gopher", "gorilla", "gosling", "gossip", "governance", "government", "governor", "gown", "grace", "grade", +"gradient", "graduate", "graduation", "graffiti", "graft", "grain", "gram", "grammar", "gran", "grand", "grandchild", "granddaughter", +"grandfather", "grandma", "grandmom", "grandmother", "grandpa", "grandparent", "grandson", "granny", "granola", "grant", "grape", "grapefruit", +"graph", "graphic", "grasp", "grass", "grasshopper", "grassland", "gratitude", "gravel", "gravitas", "gravity", "gravy", "gray", "grease", +"greatness", "greed", "green", "greenhouse", "greens", "grenade", "grey", "grid", "grief", +"grill", "grin", "grip", "gripper", "grit", "grocery", "ground", "group", "grouper", "grouse", "grove", "growth", "grub", "guacamole", +"guarantee", "guard", "guava", "guerrilla", "guess", "guest", "guestbook", "guidance", "guide", "guideline", "guilder", "guilt", "guilty", +"guinea", "guitar", "guitarist", "gum", "gumshoe", "gun", "gunpowder", "gutter", "guy", "gym", "gymnast", "gymnastics", "gynaecology", +"gyro", "habit", "habitat", "hacienda", "hacksaw", "hackwork", "hail", "hair", "haircut", "hake", "half", "half-sister", +"halibut", "hall", "halloween", "hallway", "halt", "ham", "hamburger", "hammer", "hammock", "hamster", "hand", "handball", +"handful", "handgun", "handicap", "handle", "handlebar", "handmaiden", "handover", "handrail", "handsaw", "hanger", "happening", "happiness", +"harald", "harbor", "harbour", "hardboard", "hardcover", "hardening", "hardhat", "hardship", "hardware", "hare", "harm", +"harmonica", "harmonise", "harmonize", "harmony", "harp", "harpooner", "harpsichord", "harvest", "harvester", "hash", "hashtag", "hassock", +"haste", "hat", "hatbox", "hatchet", "hatchling", "hate", "hatred", "haunt", "haven", "haversack", "havoc", "hawk", "hay", "haze", "hazel", +"hazelnut", "head", "headache", "headlight", "headline", "headphones", "headquarters", "headrest", "health", "hearing", +"hearsay", "heart", "heartache", "heartbeat", "hearth", "hearthside", "heartwood", "heat", "heater", "heating", "heaven", +"heavy", "hectare", "hedge", "hedgehog", "heel", "heifer", "height", "heir", "heirloom", "helicopter", "helium", "hell", "hellcat", "hello", +"helmet", "helo", "help", "hemisphere", "hemp", "hen", "hepatitis", "herb", "herbs", "heritage", "hermit", "hero", "heroine", "heron", +"herring", "hesitation", "heterosexual", "hexagon", "heyday", "hiccups", "hide", "hierarchy", "high", "highland", "highlight", +"highway", "hike", "hiking", "hill", "hint", "hip", "hippodrome", "hippopotamus", "hire", "hiring", "historian", "history", "hit", "hive", +"hobbit", "hobby", "hockey", "hoe", "hog", "hold", "holder", "hole", "holiday", "home", "homeland", "homeownership", "hometown", "homework", +"homicide", "homogenate", "homonym", "homosexual", "homosexuality", "honesty", "honey", "honeybee", "honeydew", "honor", "honoree", "hood", +"hoof", "hook", "hop", "hope", "hops", "horde", "horizon", "hormone", "horn", "hornet", "horror", "horse", "horseradish", "horst", "hose", +"hosiery", "hospice", "hospital", "hospitalisation", "hospitality", "hospitalization", "host", "hostel", "hostess", "hotdog", "hotel", +"hound", "hour", "hourglass", "house", "houseboat", "household", "housewife", "housework", "housing", "hovel", "hovercraft", "howard", +"howitzer", "hub", "hubcap", "hubris", "hug", "hugger", "hull", "human", "humanity", "humidity", "hummus", "humor", "humour", "hunchback", +"hundred", "hunger", "hunt", "hunter", "hunting", "hurdle", "hurdler", "hurricane", "hurry", "hurt", "husband", "hut", "hutch", "hyacinth", +"hybridisation", "hybridization", "hydrant", "hydraulics", "hydrocarb", "hydrocarbon", "hydrofoil", "hydrogen", "hydrolyse", "hydrolysis", +"hydrolyze", "hydroxyl", "hyena", "hygienic", "hype", "hyphenation", "hypochondria", "hypothermia", "hypothesis", "ice", "ice-cream", +"iceberg", "icebreaker", "icecream", "icicle", "icing", "icon", "icy", "id", "idea", "ideal", "identification", "identity", "ideology", +"idiom", "idiot", "igloo", "ignorance", "ignorant", "ikebana", "illegal", "illiteracy", "illness", "illusion", "illustration", "image", +"imagination", "imbalance", "imitation", "immigrant", "immigration", "immortal", "impact", "impairment", "impala", "impediment", "implement", +"implementation", "implication", "import", "importance", "impostor", "impress", "impression", "imprisonment", "impropriety", "improvement", +"impudence", "impulse", "inability", "inauguration", "inbox", "incandescence", "incarnation", "incense", "incentive", +"inch", "incidence", "incident", "incision", "inclusion", "income", "incompetence", "inconvenience", "increase", "incubation", "independence", +"independent", "index", "indication", "indicator", "indigence", "individual", "industrialisation", "industrialization", "industry", "inequality", +"inevitable", "infancy", "infant", "infarction", "infection", "infiltration", "infinite", "infix", "inflammation", "inflation", "influence", +"influx", "info", "information", "infrastructure", "infusion", "inglenook", "ingrate", "ingredient", "inhabitant", "inheritance", "inhibition", +"inhibitor", "initial", "initialise", "initialize", "initiative", "injunction", "injury", "injustice", "ink", "inlay", "inn", "innervation", +"innocence", "innocent", "innovation", "input", "inquiry", "inscription", "insect", "insectarium", "insert", "inside", "insight", "insolence", +"insomnia", "inspection", "inspector", "inspiration", "installation", "instance", "instant", "instinct", "institute", "institution", +"instruction", "instructor", "instrument", "instrumentalist", "instrumentation", "insulation", "insurance", "insurgence", "insurrection", +"integer", "integral", "integration", "integrity", "intellect", "intelligence", "intensity", "intent", "intention", "intentionality", +"interaction", "interchange", "interconnection", "intercourse", "interest", "interface", "interferometer", "interior", "interject", "interloper", +"internet", "interpretation", "interpreter", "interval", "intervenor", "intervention", "interview", "interviewer", "intestine", "introduction", +"intuition", "invader", "invasion", "invention", "inventor", "inventory", "inverse", "inversion", "investigation", "investigator", "investment", +"investor", "invitation", "invite", "invoice", "involvement", "iridescence", "iris", "iron", "ironclad", "irony", "irrigation", "ischemia", +"island", "isogloss", "isolation", "issue", "item", "itinerary", "ivory", "jack", "jackal", "jacket", "jackfruit", "jade", "jaguar", +"jail", "jailhouse", "jalapeño", "jam", "jar", "jasmine", "jaw", "jazz", "jealousy", "jeans", "jeep", "jelly", "jellybeans", "jellyfish", +"jerk", "jet", "jewel", "jeweller", "jewellery", "jewelry", "jicama", "jiffy", "job", "jockey", "jodhpurs", "joey", "jogging", "joint", +"joke", "jot", "journal", "journalism", "journalist", "journey", "joy", "judge", "judgment", "judo", "jug", "juggernaut", "juice", "julienne", +"jumbo", "jump", "jumper", "jumpsuit", "jungle", "junior", "junk", "junker", "junket", "jury", "justice", "justification", "jute", "kale", +"kamikaze", "kangaroo", "karate", "kayak", "kazoo", "kebab", "keep", "keeper", "kendo", "kennel", "ketch", "ketchup", "kettle", "kettledrum", +"key", "keyboard", "keyboarding", "keystone", "kick", "kid", "kidney", "kielbasa", "kill", "killer", "killing", "kilogram", +"kilometer", "kilt", "kimono", "kinase", "kind", "kindness", "king", "kingdom", "kingfish", "kiosk", "kiss", "kit", "kitchen", "kite", +"kitsch", "kitten", "kitty", "kiwi", "knee", "kneejerk", "knickers", "knife", "knight", "knitting", "knock", "knot", "know-how", +"knowledge", "knuckle", "koala", "kohlrabi", "kumquat", "lab", "label", "labor", "laboratory", "laborer", "labour", "labourer", "lace", +"lack", "lacquerware", "lad", "ladder", "ladle", "lady", "ladybug", "lag", "lake", "lamb", "lambkin", "lament", "lamp", "lanai", "land", +"landform", "landing", "landmine", "landscape", "lane", "language", "lantern", "lap", "laparoscope", "lapdog", "laptop", "larch", "lard", +"larder", "lark", "larva", "laryngitis", "lasagna", "lashes", "last", "latency", "latex", "lathe", "latitude", "latte", "latter", "laugh", +"laughter", "laundry", "lava", "law", "lawmaker", "lawn", "lawsuit", "lawyer", "lay", "layer", "layout", "lead", "leader", "leadership", +"leading", "leaf", "league", "leaker", "leap", "learning", "leash", "leather", "leave", "leaver", "lecture", "leek", "leeway", "left", +"leg", "legacy", "legal", "legend", "legging", "legislation", "legislator", "legislature", "legitimacy", "legume", "leisure", "lemon", +"lemonade", "lemur", "lender", "lending", "length", "lens", "lentil", "leopard", "leprosy", "leptocephalus", "lesbian", "lesson", "letter", +"lettuce", "level", "lever", "leverage", "leveret", "liability", "liar", "liberty", "libido", "library", "licence", "license", "licensing", +"licorice", "lid", "lie", "lieu", "lieutenant", "life", "lifestyle", "lifetime", "lift", "ligand", "light", "lighting", "lightning", +"lightscreen", "ligula", "likelihood", "likeness", "lilac", "lily", "limb", "lime", "limestone", "limit", "limitation", "limo", "line", +"linen", "liner", "linguist", "linguistics", "lining", "link", "linkage", "linseed", "lion", "lip", "lipid", "lipoprotein", "lipstick", +"liquid", "liquidity", "liquor", "list", "listening", "listing", "literate", "literature", "litigation", "litmus", "litter", "littleneck", +"liver", "livestock", "living", "lizard", "llama", "load", "loading", "loaf", "loafer", "loan", "lobby", "lobotomy", "lobster", "local", +"locality", "location", "lock", "locker", "locket", "locomotive", "locust", "lode", "loft", "log", "loggia", "logic", "login", "logistics", +"logo", "loincloth", "lollipop", "loneliness", "longboat", "longitude", "look", "lookout", "loop", "loophole", "loquat", "lord", "loss", +"lot", "lotion", "lottery", "lounge", "louse", "lout", "love", "lover", "lox", "loyalty", "luck", "luggage", "lumber", "lumberman", "lunch", +"luncheonette", "lunchmeat", "lunchroom", "lung", "lunge", "lust", "lute", "luxury", "lychee", "lycra", "lye", "lymphocyte", "lynx", +"lyocell", "lyre", "lyrics", "lysine", "mRNA", "macadamia", "macaroni", "macaroon", "macaw", "machine", "machinery", "macrame", "macro", +"macrofauna", "madam", "maelstrom", "maestro", "magazine", "maggot", "magic", "magnet", "magnitude", "maid", "maiden", "mail", "mailbox", +"mailer", "mailing", "mailman", "main", "mainland", "mainstream", "maintainer", "maintenance", "maize", "major", "majority", +"makeover", "maker", "makeup", "making", "male", "malice", "mall", "mallard", "mallet", "malnutrition", "mama", "mambo", "mammoth", "man", +"manacle", "management", "manager", "manatee", "mandarin", "mandate", "mandolin", "mangle", "mango", "mangrove", "manhunt", "maniac", +"manicure", "manifestation", "manipulation", "mankind", "manner", "manor", "mansard", "manservant", "mansion", "mantel", "mantle", "mantua", +"manufacturer", "manufacturing", "many", "map", "maple", "mapping", "maracas", "marathon", "marble", "march", "mare", "margarine", "margin", +"mariachi", "marimba", "marines", "marionberry", "mark", "marker", "market", "marketer", "marketing", "marketplace", "marksman", "markup", +"marmalade", "marriage", "marsh", "marshland", "marshmallow", "marten", "marxism", "mascara", "mask", "masonry", "mass", "massage", "mast", +"master", "masterpiece", "mastication", "mastoid", "mat", "match", "matchmaker", "mate", "material", "maternity", "math", "mathematics", +"matrix", "matter", "mattock", "mattress", "max", "maximum", "maybe", "mayonnaise", "mayor", "meadow", "meal", "mean", "meander", "meaning", +"means", "meantime", "measles", "measure", "measurement", "meat", "meatball", "meatloaf", "mecca", "mechanic", "mechanism", "med", "medal", +"media", "median", "medication", "medicine", "medium", "meet", "meeting", "melatonin", "melody", "melon", "member", "membership", "membrane", +"meme", "memo", "memorial", "memory", "men", "menopause", "menorah", "mention", "mentor", "menu", "merchandise", "merchant", "mercury", +"meridian", "meringue", "merit", "mesenchyme", "mess", "message", "messenger", "messy", "metabolite", "metal", "metallurgist", "metaphor", +"meteor", "meteorology", "meter", "methane", "method", "methodology", "metric", "metro", "metronome", "mezzanine", "microlending", "micronutrient", +"microphone", "microwave", "midden", "middle", "middleman", "midline", "midnight", "midwife", "might", "migrant", "migration", +"mile", "mileage", "milepost", "milestone", "military", "milk", "milkshake", "mill", "millennium", "millet", "millimeter", "million", +"millisecond", "millstone", "mime", "mimosa", "min", "mincemeat", "mind", "mine", "mineral", "mineshaft", "mini", "minibus", +"minimalism", "minimum", "mining", "minion", "minister", "mink", "minnow", "minor", "minority", "mint", "minute", "miracle", +"mirror", "miscarriage", "miscommunication", "misfit", "misnomer", "misogyny", "misplacement", "misreading", "misrepresentation", "miss", +"missile", "mission", "missionary", "mist", "mistake", "mister", "misunderstand", "miter", "mitten", "mix", "mixer", "mixture", "moai", +"moat", "mob", "mobile", "mobility", "mobster", "moccasins", "mocha", "mochi", "mode", "model", "modeling", "modem", "modernist", "modernity", +"modification", "molar", "molasses", "molding", "mole", "molecule", "mom", "moment", "monastery", "monasticism", "money", "monger", "monitor", +"monitoring", "monk", "monkey", "monocle", "monopoly", "monotheism", "monsoon", "monster", "month", "monument", "mood", "moody", "moon", +"moonlight", "moonscape", "moonshine", "moose", "mop", "morale", "morbid", "morbidity", "morning", "moron", "morphology", "morsel", "mortal", +"mortality", "mortgage", "mortise", "mosque", "mosquito", "most", "motel", "moth", "mother", "motion", "motivation", +"motive", "motor", "motorboat", "motorcar", "motorcycle", "mound", "mountain", "mouse", "mouser", "mousse", "moustache", "mouth", "mouton", +"movement", "mover", "movie", "mower", "mozzarella", "mud", "muffin", "mug", "mukluk", "mule", "multimedia", "murder", "muscat", "muscatel", +"muscle", "musculature", "museum", "mushroom", "music", "musician", "muskrat", "mussel", "mustache", "mustard", +"mutation", "mutt", "mutton", "mycoplasma", "mystery", "myth", "mythology", "nail", "name", "naming", "nanoparticle", "napkin", "narrative", +"nasal", "nation", "nationality", "native", "naturalisation", "nature", "navigation", "necessity", "neck", "necklace", "necktie", "nectar", +"nectarine", "need", "needle", "neglect", "negligee", "negotiation", "neighbor", "neighborhood", "neighbour", "neighbourhood", "neologism", +"neon", "neonate", "nephew", "nerve", "nest", "nestling", "nestmate", "net", "netball", "netbook", "netsuke", "network", "networking", +"neurobiologist", "neuron", "neuropathologist", "neuropsychiatry", "news", "newsletter", "newspaper", "newsprint", "newsstand", "nexus", +"nibble", "nicety", "niche", "nick", "nickel", "nickname", "niece", "night", "nightclub", "nightgown", "nightingale", "nightlife", "nightlight", +"nightmare", "ninja", "nit", "nitrogen", "nobody", "nod", "node", "noir", "noise", "nonbeliever", "nonconformist", "nondisclosure", "nonsense", +"noodle", "noodles", "noon", "norm", "normal", "normalisation", "normalization", "north", "nose", "notation", "note", "notebook", "notepad", +"nothing", "notice", "notion", "notoriety", "nougat", "noun", "nourishment", "novel", "nucleotidase", "nucleotide", "nudge", "nuke", +"number", "numeracy", "numeric", "numismatist", "nun", "nurse", "nursery", "nursing", "nurture", "nut", "nutmeg", "nutrient", "nutrition", +"nylon", "nymph", "oak", "oar", "oasis", "oat", "oatmeal", "oats", "obedience", "obesity", "obi", "object", "objection", "objective", +"obligation", "oboe", "observation", "observatory", "obsession", "obsidian", "obstacle", "occasion", "occupation", "occurrence", "ocean", +"ocelot", "octagon", "octave", "octavo", "octet", "octopus", "odometer", "odyssey", "oeuvre", "offence", "offense", "offer", +"offering", "office", "officer", "official", "offset", "oil", "okra", "oldie", "oleo", "olive", "omega", "omelet", "omission", "omnivore", +"oncology", "onion", "online", "onset", "opening", "opera", "operating", "operation", "operator", "ophthalmologist", "opinion", "opium", +"opossum", "opponent", "opportunist", "opportunity", "opposite", "opposition", "optimal", "optimisation", "optimist", "optimization", +"option", "orange", "orangutan", "orator", "orchard", "orchestra", "orchid", "order", "ordinary", "ordination", "ore", "oregano", "organ", +"organisation", "organising", "organization", "organizing", "orient", "orientation", "origin", "original", "originality", "ornament", +"osmosis", "osprey", "ostrich", "other", "otter", "ottoman", "ounce", "outback", "outcome", "outfielder", "outfit", "outhouse", "outlaw", +"outlay", "outlet", "outline", "outlook", "output", "outrage", "outrigger", "outrun", "outset", "outside", "oval", "ovary", "oven", "overcharge", +"overclocking", "overcoat", "overexertion", "overflight", "overhead", "overheard", "overload", "overnighter", "overshoot", "oversight", +"overview", "overweight", "owl", "owner", "ownership", "ox", "oxford", "oxygen", "oyster", "ozone", "pace", "pacemaker", "pack", "package", +"packaging", "packet", "pad", "paddle", "paddock", "pagan", "page", "pagoda", "pail", "pain", "paint", "painter", "painting", "paintwork", +"pair", "pajamas", "palace", "palate", "palm", "pamphlet", "pan", "pancake", "pancreas", "panda", "panel", "panic", "pannier", "panpipe", +"pansy", "panther", "panties", "pantologist", "pantology", "pantry", "pants", "pantsuit", "panty", "pantyhose", "papa", "papaya", "paper", +"paperback", "paperwork", "parable", "parachute", "parade", "paradise", "paragraph", "parallelogram", "paramecium", "paramedic", "parameter", +"paranoia", "parcel", "parchment", "pard", "pardon", "parent", "parenthesis", "parenting", "park", "parka", "parking", "parliament", +"parole", "parrot", "parser", "parsley", "parsnip", "part", "participant", "participation", "particle", "particular", "partner", "partnership", +"partridge", "party", "pass", "passage", "passbook", "passenger", "passing", "passion", "passive", "passport", "password", "past", "pasta", +"paste", "pastor", "pastoralist", "pastry", "pasture", "pat", "patch", "pate", "patent", "patentee", "path", "pathogenesis", "pathology", +"pathway", "patience", "patient", "patina", "patio", "patriarch", "patrimony", "patriot", "patrol", "patroller", "patrolling", "patron", +"pattern", "patty", "pattypan", "pause", "pavement", "pavilion", "paw", "pawnshop", "pay", "payee", "payment", "payoff", "pea", "peace", +"peach", "peacoat", "peacock", "peak", "peanut", "pear", "pearl", "peasant", "pecan", "pecker", "pedal", "peek", "peen", "peer", "peer-to-peer", +"pegboard", "pelican", "pelt", "pen", "penalty", "pence", "pencil", "pendant", "pendulum", "penguin", "penicillin", "peninsula", "penis", +"pennant", "penny", "pension", "pentagon", "peony", "people", "pepper", "pepperoni", "percent", "percentage", "perception", "perch", +"perennial", "perfection", "performance", "perfume", "period", "periodical", "peripheral", "permafrost", "permission", "permit", "perp", +"perpendicular", "persimmon", "person", "personal", "personality", "personnel", "perspective", "pest", "pet", "petal", "petition", "petitioner", +"petticoat", "pew", "pharmacist", "pharmacopoeia", "phase", "pheasant", "phenomenon", "phenotype", "pheromone", "philanthropy", "philosopher", +"philosophy", "phone", "phosphate", "photo", "photodiode", "photograph", "photographer", "photography", "photoreceptor", "phrase", "phrasing", +"physical", "physics", "physiology", "pianist", "piano", "piccolo", "pick", "pickax", "pickaxe", "picket", "pickle", "pickup", "picnic", +"picture", "picturesque", "pie", "piece", "pier", "piety", "pig", "pigeon", "piglet", "pigpen", "pigsty", "pike", "pilaf", "pile", "pilgrim", +"pilgrimage", "pill", "pillar", "pillbox", "pillow", "pilot", "pimp", "pimple", "pin", "pinafore", "pine", "pineapple", +"pinecone", "ping", "pink", "pinkie", "pinot", "pinstripe", "pint", "pinto", "pinworm", "pioneer", "pipe", "pipeline", "piracy", "pirate", +"piss", "pistol", "pit", "pita", "pitch", "pitcher", "pitching", "pith", "pizza", "place", "placebo", "placement", "placode", "plagiarism", +"plain", "plaintiff", "plan", "plane", "planet", "planning", "plant", "plantation", "planter", "planula", "plaster", "plasterboard", +"plastic", "plate", "platelet", "platform", "platinum", "platter", "platypus", "play", "player", "playground", "playroom", "playwright", +"plea", "pleasure", "pleat", "pledge", "plenty", "plier", "pliers", "plight", "plot", "plough", "plover", "plow", "plowman", "plug", +"plugin", "plum", "plumber", "plume", "plunger", "plywood", "pneumonia", "pocket", "pocketbook", "pod", "podcast", "poem", +"poet", "poetry", "poignance", "point", "poison", "poisoning", "poker", "polarisation", "polarization", "pole", "polenta", "police", +"policeman", "policy", "polish", "politician", "politics", "poll", "polliwog", "pollutant", "pollution", "polo", "polyester", "polyp", +"pomegranate", "pomelo", "pompom", "poncho", "pond", "pony", "pool", "poor", "pop", "popcorn", "poppy", "popsicle", "popularity", "population", +"populist", "porcelain", "porch", "porcupine", "pork", "porpoise", "port", "porter", "portfolio", "porthole", "portion", "portrait", +"position", "possession", "possibility", "possible", "post", "postage", "postbox", "poster", "posterior", "postfix", "pot", "potato", +"potential", "pottery", "potty", "pouch", "poultry", "pound", "pounding", "poverty", "powder", "power", "practice", "practitioner", "prairie", +"praise", "pray", "prayer", "precedence", "precedent", "precipitation", "precision", "predecessor", "preface", "preference", "prefix", +"pregnancy", "prejudice", "prelude", "premeditation", "premier", "premise", "premium", "preoccupation", "preparation", "prescription", +"presence", "present", "presentation", "preservation", "preserves", "presidency", "president", "press", "pressroom", "pressure", "pressurisation", +"pressurization", "prestige", "presume", "pretzel", "prevalence", "prevention", "prey", "price", "pricing", "pride", "priest", "priesthood", +"primary", "primate", "prince", "princess", "principal", "principle", "print", "printer", "printing", "prior", "priority", "prison", +"prisoner", "privacy", "private", "privilege", "prize", "prizefight", "probability", "probation", "probe", "problem", "procedure", "proceedings", +"process", "processing", "processor", "proctor", "procurement", "produce", "producer", "product", "production", "productivity", "profession", +"professional", "professor", "profile", "profit", "progenitor", "program", "programme", "programming", "progress", "progression", "prohibition", +"project", "proliferation", "promenade", "promise", "promotion", "prompt", "pronoun", "pronunciation", "proof", "propaganda", +"propane", "property", "prophet", "proponent", "proportion", "proposal", "proposition", "proprietor", "prose", "prosecution", "prosecutor", +"prospect", "prosperity", "prostacyclin", "prostanoid", "prostrate", "protection", "protein", "protest", "protocol", "providence", "provider", +"province", "provision", "prow", "proximal", "proximity", "prune", "pruner", "pseudocode", "pseudoscience", "psychiatrist", "psychoanalyst", +"psychologist", "psychology", "ptarmigan", "pub", "public", "publication", "publicity", "publisher", "publishing", "pudding", "puddle", +"puffin", "pug", "puggle", "pulley", "pulse", "puma", "pump", "pumpernickel", "pumpkin", "pumpkinseed", "pun", "punch", "punctuation", +"punishment", "pup", "pupa", "pupil", "puppet", "puppy", "purchase", "puritan", "purity", "purple", "purpose", "purr", "purse", "pursuit", +"push", "pusher", "put", "puzzle", "pyramid", "pyridine", "quadrant", "quail", "qualification", "quality", "quantity", "quart", "quarter", +"quartet", "quartz", "queen", "query", "quest", "question", "questioner", "questionnaire", "quiche", "quicksand", "quiet", "quill", "quilt", +"quince", "quinoa", "quit", "quiver", "quota", "quotation", "quote", "rabbi", "rabbit", "raccoon", "race", "racer", "racing", "racism", +"racist", "rack", "radar", "radiator", "radio", "radiosonde", "radish", "raffle", "raft", "rag", "rage", "raid", "rail", "railing", "railroad", +"railway", "raiment", "rain", "rainbow", "raincoat", "rainmaker", "rainstorm", "rainy", "raise", "raisin", "rake", "rally", "ram", "rambler", +"ramen", "ramie", "ranch", "rancher", "randomisation", "randomization", "range", "ranger", "rank", "rap", "rape", "raspberry", "rat", +"rate", "ratepayer", "rating", "ratio", "rationale", "rations", "raven", "ravioli", "rawhide", "ray", "rayon", "razor", "reach", "reactant", +"reaction", "read", "reader", "readiness", "reading", "real", "reality", "realization", "realm", "reamer", "rear", "reason", "reasoning", +"rebel", "rebellion", "reboot", "recall", "recapitulation", "receipt", "receiver", "reception", "receptor", "recess", "recession", "recipe", +"recipient", "reciprocity", "reclamation", "recliner", "recognition", "recollection", "recommendation", "reconsideration", "record", +"recorder", "recording", "recovery", "recreation", "recruit", "rectangle", "red", "redesign", "redhead", "redirect", "rediscovery", "reduction", +"reef", "refectory", "reference", "referendum", "reflection", "reform", "refreshments", "refrigerator", "refuge", "refund", "refusal", +"refuse", "regard", "regime", "region", "regionalism", "register", "registration", "registry", "regret", "regulation", "regulator", "rehospitalisation", +"rehospitalization", "reindeer", "reinscription", "reject", "relation", "relationship", "relative", "relaxation", "relay", "release", +"reliability", "relief", "religion", "relish", "reluctance", "remains", "remark", "reminder", "remnant", "remote", "removal", "renaissance", +"rent", "reorganisation", "reorganization", "repair", "reparation", "repayment", "repeat", "replacement", "replica", "replication", "reply", +"report", "reporter", "reporting", "repository", "representation", "representative", "reprocessing", "republic", "republican", "reputation", +"request", "requirement", "resale", "rescue", "research", "researcher", "resemblance", "reservation", "reserve", "reservoir", "reset", +"residence", "resident", "residue", "resist", "resistance", "resolution", "resolve", "resort", "resource", "respect", "respite", "response", +"responsibility", "rest", "restaurant", "restoration", "restriction", "restroom", "restructuring", "result", "resume", "retailer", "retention", +"rethinking", "retina", "retirement", "retouching", "retreat", "retrospect", "retrospective", "retrospectivity", "return", "reunion", +"revascularisation", "revascularization", "reveal", "revelation", "revenant", "revenge", "revenue", "reversal", "reverse", "review", +"revitalisation", "revitalization", "revival", "revolution", "revolver", "reward", "rhetoric", "rheumatism", "rhinoceros", "rhubarb", +"rhyme", "rhythm", "rib", "ribbon", "rice", "riddle", "ride", "rider", "ridge", "riding", "rifle", "right", "rim", "ring", "ringworm", +"riot", "rip", "ripple", "rise", "riser", "risk", "rite", "ritual", "river", "riverbed", "rivulet", "road", "roadway", "roar", "roast", +"robe", "robin", "robot", "robotics", "rock", "rocker", "rocket", "rod", "role", "roll", "roller", "romaine", "romance", +"roof", "room", "roommate", "rooster", "root", "rope", "rose", "rosemary", "roster", "rostrum", "rotation", "round", "roundabout", "route", +"router", "routine", "row", "rowboat", "rowing", "rubber", "rubbish", "rubric", "ruby", "ruckus", "rudiment", "ruffle", "rug", "rugby", +"ruin", "rule", "ruler", "ruling", "rum", "rumor", "run", "runaway", "runner", "running", "runway", "rush", "rust", "rutabaga", "rye", +"sabre", "sac", "sack", "saddle", "sadness", "safari", "safe", "safeguard", "safety", "saffron", "sage", "sail", "sailboat", "sailing", +"sailor", "saint", "sake", "salad", "salami", "salary", "sale", "salesman", "salmon", "salon", "saloon", "salsa", "salt", "salute", "samovar", +"sampan", "sample", "samurai", "sanction", "sanctity", "sanctuary", "sand", "sandal", "sandbar", "sandpaper", "sandwich", "sanity", "sardine", +"sari", "sarong", "sash", "satellite", "satin", "satire", "satisfaction", "sauce", "saucer", "sauerkraut", "sausage", "savage", "savannah", +"saving", "savings", "savior", "saviour", "savory", "saw", "saxophone", "scaffold", "scale", "scallion", "scallops", "scalp", "scam", +"scanner", "scarecrow", "scarf", "scarification", "scenario", "scene", "scenery", "scent", "schedule", "scheduling", "schema", "scheme", +"schizophrenic", "schnitzel", "scholar", "scholarship", "school", "schoolhouse", "schooner", "science", "scientist", "scimitar", "scissors", +"scooter", "scope", "score", "scorn", "scorpion", "scotch", "scout", "scow", "scrambled", "scrap", "scraper", "scratch", "screamer", +"screen", "screening", "screenwriting", "screw", "screwdriver", "scrim", "scrip", "script", "scripture", "scrutiny", "sculpting", +"sculptural", "sculpture", "sea", "seabass", "seafood", "seagull", "seal", "seaplane", "search", "seashore", "seaside", "season", "seat", +"seaweed", "second", "secrecy", "secret", "secretariat", "secretary", "secretion", "section", "sectional", "sector", "security", "sediment", +"seed", "seeder", "seeker", "seep", "segment", "seizure", "selection", "self", "seller", +"selling", "semantics", "semester", "semicircle", "semicolon", "semiconductor", "seminar", "senate", "senator", "sender", "senior", "sense", +"sensibility", "sensitive", "sensitivity", "sensor", "sentence", "sentencing", "sentiment", "sepal", "separation", "septicaemia", "sequel", +"sequence", "serial", "series", "sermon", "serum", "serval", "servant", "server", "service", "servitude", "sesame", "session", "set", +"setback", "setting", "settlement", "settler", "severity", "sewer", "sex", "sexuality", "shack", "shackle", "shade", "shadow", "shadowbox", +"shakedown", "shaker", "shallot", "shallows", "shame", "shampoo", "shanty", "shape", "share", "shareholder", "shark", "shaw", "shawl", +"shear", "shearling", "sheath", "shed", "sheep", "sheet", "shelf", "shell", "shelter", "sherbet", "sherry", "shield", "shift", "shin", +"shine", "shingle", "ship", "shipper", "shipping", "shipyard", "shirt", "shirtdress", "shit", "shoat", "shock", "shoe", "shoe-horn", +"shoehorn", "shoelace", "shoemaker", "shoes", "shoestring", "shofar", "shoot", "shootdown", "shop", "shopper", "shopping", "shore", "shoreline", +"short", "shortage", "shorts", "shortwave", "shot", "shoulder", "shout", "shovel", "show", "shower", "shred", "shrimp", +"shrine", "shutdown", "sibling", "sick", "sickness", "side", "sideboard", "sideburns", "sidecar", "sidestream", "sidewalk", "siding", +"siege", "sigh", "sight", "sightseeing", "sign", "signal", "signature", "signet", "significance", "signify", "signup", "silence", "silica", +"silicon", "silk", "silkworm", "sill", "silly", "silo", "silver", "similarity", "simple", "simplicity", "simplification", "simvastatin", +"sin", "singer", "singing", "singular", "sink", "sinuosity", "sip", "sir", "sister", "sitar", "site", "situation", "size", +"skate", "skating", "skean", "skeleton", "ski", "skiing", "skill", "skin", "skirt", "skull", "skullcap", "skullduggery", "skunk", "sky", +"skylight", "skyline", "skyscraper", "skywalk", "slang", "slapstick", "slash", "slate", "slave", "slavery", "slaw", "sled", "sledge", +"sleep", "sleepiness", "sleeping", "sleet", "sleuth", "slice", "slide", "slider", "slime", "slip", "slipper", "slippers", "slope", "slot", +"sloth", "slump", "smell", "smelting", "smile", "smith", "smock", "smog", "smoke", "smoking", "smolt", "smuggling", "snack", "snail", +"snake", "snakebite", "snap", "snarl", "sneaker", "sneakers", "sneeze", "sniffle", "snob", "snorer", "snow", "snowboarding", "snowflake", +"snowman", "snowmobiling", "snowplow", "snowstorm", "snowsuit", "snuck", "snug", "snuggle", "soap", "soccer", "socialism", "socialist", +"society", "sociology", "sock", "socks", "soda", "sofa", "softball", "softdrink", "softening", "software", "soil", "soldier", "sole", +"solicitation", "solicitor", "solidarity", "solidity", "soliloquy", "solitaire", "solution", "solvency", "sombrero", "somebody", "someone", +"someplace", "somersault", "something", "somewhere", "son", "sonar", "sonata", "song", "songbird", "sonnet", "soot", "sophomore", "soprano", +"sorbet", "sorghum", "sorrel", "sorrow", "sort", "soul", "soulmate", "sound", "soundness", "soup", "source", "sourwood", "sousaphone", +"south", "southeast", "souvenir", "sovereignty", "sow", "soy", "soybean", "space", "spacing", "spade", "spaghetti", "span", "spandex", +"spank", "sparerib", "spark", "sparrow", "spasm", "spat", "spatula", "spawn", "speaker", "speakerphone", "speaking", "spear", "spec", +"special", "specialist", "specialty", "species", "specification", "spectacle", "spectacles", "spectrograph", "spectrum", "speculation", +"speech", "speed", "speedboat", "spell", "spelling", "spelt", "spending", "sphere", "sphynx", "spice", "spider", "spiderling", "spike", +"spill", "spinach", "spine", "spiral", "spirit", "spiritual", "spirituality", "spit", "spite", "spleen", "splendor", "split", "spokesman", +"spokeswoman", "sponge", "sponsor", "sponsorship", "spool", "spoon", "spork", "sport", "sportsman", "spot", "spotlight", "spouse", "sprag", +"sprat", "spray", "spread", "spreadsheet", "spree", "spring", "sprinkles", "sprinter", "sprout", "spruce", "spud", "spume", "spur", "spy", +"spyglass", "square", "squash", "squatter", "squeegee", "squid", "squirrel", "stab", "stability", "stable", "stack", "stacking", "stadium", +"staff", "stag", "stage", "stain", "stair", "staircase", "stake", "stalk", "stall", "stallion", "stamen", "stamina", "stamp", "stance", +"stand", "standard", "standardisation", "standardization", "standing", "standoff", "standpoint", "star", "starboard", "start", "starter", +"state", "statement", "statin", "station", "statistic", "statistics", "statue", "status", "statute", "stay", "steak", +"stealth", "steam", "steamroller", "steel", "steeple", "stem", "stench", "stencil", "step", "step-daughter", +"stepdaughter", "stepmother", +"stepson", "stereo", "stew", "steward", "stick", "sticker", "stiletto", "still", "stimulation", "stimulus", "sting", +"stinger", "stitch", "stitcher", "stock", "stockings", "stole", "stomach", "stone", "stonework", "stool", +"stop", "stopsign", "stopwatch", "storage", "store", "storey", "storm", "story", "storyboard", "stot", "stove", "strait", +"strand", "stranger", "strap", "strategy", "straw", "strawberry", "strawman", "stream", "street", "streetcar", "strength", "stress", +"stretch", "strife", "strike", "string", "strip", "stripe", "strobe", "stroke", "structure", "strudel", "struggle", "stucco", "stud", +"student", "studio", "study", "stuff", "stumbling", "stump", "stupidity", "sturgeon", "sty", "style", "styling", "stylus", "sub", "subcomponent", +"subconscious", "subcontractor", "subexpression", "subgroup", "subject", "submarine", "submitter", "subprime", "subroutine", "subscription", +"subsection", "subset", "subsidence", "subsidiary", "subsidy", "substance", "substitution", "subtitle", "suburb", "subway", "success", +"succotash", "suck", "sucker", "suede", "suet", "suffocation", "sugar", "suggestion", "suicide", "suit", "suitcase", "suite", "sulfur", +"sultan", "sum", "summary", "summer", "summit", "sun", "sunbeam", "sunbonnet", "sundae", "sunday", "sundial", "sunflower", "sunglasses", +"sunlamp", "sunlight", "sunrise", "sunroom", "sunset", "sunshine", "superiority", "supermarket", "supernatural", "supervision", "supervisor", +"supper", "supplement", "supplier", "supply", "support", "supporter", "suppression", "supreme", "surface", "surfboard", "surge", "surgeon", +"surgery", "surname", "surplus", "surprise", "surround", "surroundings", "surrounds", "survey", "survival", "survivor", "sushi", "suspect", +"suspenders", "suspension", "sustainment", "sustenance", "swallow", "swamp", "swan", "swanling", "swath", "sweat", "sweater", "sweatshirt", +"sweatshop", "sweatsuit", "sweets", "swell", "swim", "swimming", "swimsuit", "swine", "swing", "switch", "switchboard", "switching", +"swivel", "sword", "swordfight", "swordfish", "sycamore", "symbol", "symmetry", "sympathy", "symptom", "syndicate", "syndrome", "synergy", +"synod", "synonym", "synthesis", "syrup", "system", "tab", "tabby", "tabernacle", "table", "tablecloth", "tablet", "tabletop", +"tachometer", "tackle", "taco", "tactics", "tactile", "tadpole", "tag", "tail", "tailbud", "tailor", "tailspin", "takeover", +"tale", "talent", "talk", "talking", "tamale", "tambour", "tambourine", "tan", "tandem", "tangerine", "tank", "tank-top", +"tanker", "tankful", "tap", "tape", "tapioca", "target", "taro", "tarragon", "tart", "task", "tassel", "taste", "tatami", "tattler", +"tattoo", "tavern", "tax", "taxi", "taxicab", "taxpayer", "tea", "teacher", "teaching", "team", "teammate", "teapot", "tear", "tech", +"technician", "technique", "technologist", "technology", "tectonics", "teen", "teenager", "teepee", "telephone", "telescreen", "teletype", +"television", "tell", "teller", "temp", "temper", "temperature", "temple", "tempo", "temporariness", "temporary", "temptation", "temptress", +"tenant", "tendency", "tender", "tenement", "tenet", "tennis", "tenor", "tension", "tensor", "tent", "tentacle", "tenth", "tepee", "teriyaki", +"term", "terminal", "termination", "terminology", "termite", "terrace", "terracotta", "terrapin", "terrarium", "territory", "terror", +"terrorism", "terrorist", "test", "testament", "testimonial", "testimony", "testing", "text", "textbook", "textual", "texture", "thanks", +"thaw", "theater", "theft", "theism", "theme", "theology", "theory", "therapist", "therapy", "thermals", "thermometer", "thermostat", +"thesis", "thickness", "thief", "thigh", "thing", "thinking", "thirst", "thistle", "thong", "thongs", "thorn", "thought", "thousand", +"thread", "threat", "threshold", "thrift", "thrill", "throat", "throne", "thrush", "thrust", "thug", "thumb", "thump", "thunder", "thunderbolt", +"thunderhead", "thunderstorm", "thyme", "tiara", "tic", "tick", "ticket", "tide", "tie", "tiger", "tights", "tile", "till", "tilt", "timbale", +"timber", "time", "timeline", "timeout", "timer", "timetable", "timing", "timpani", "tin", "tinderbox", "tinkle", "tintype", "tip", "tire", +"tissue", "titanium", "title", "toad", "toast", "toaster", "tobacco", "today", "toe", "toenail", "toffee", "tofu", "tog", "toga", "toilet", +"tolerance", "tolerant", "toll", "tomatillo", "tomato", "tomb", "tomography", "tomorrow", "ton", "tonality", "tone", "tongue", +"tonic", "tonight", "tool", "toot", "tooth", "toothbrush", "toothpaste", "toothpick", "top", "topic", "topsail", "toque", +"toreador", "tornado", "torso", "torte", "tortellini", "tortilla", "tortoise", "tosser", "total", "tote", "touch", "tour", +"tourism", "tourist", "tournament", "towel", "tower", "town", "townhouse", "township", "toy", "trace", "trachoma", "track", +"tracking", "tracksuit", "tract", "tractor", "trade", "trader", "trading", "tradition", "traditionalism", "traffic", "trafficker", "tragedy", +"trail", "trailer", "trailpatrol", "train", "trainer", "training", "trait", "tram", "tramp", "trance", "transaction", "transcript", "transfer", +"transformation", "transit", "transition", "translation", "transmission", "transom", "transparency", "transplantation", "transport", +"transportation", "trap", "trapdoor", "trapezium", "trapezoid", "trash", "travel", "traveler", "tray", "treasure", "treasury", "treat", +"treatment", "treaty", "tree", "trek", "trellis", "tremor", "trench", "trend", "triad", "trial", "triangle", "tribe", "tributary", "trick", +"trigger", "trigonometry", "trillion", "trim", "trinket", "trip", "tripod", "tritone", "triumph", "trolley", "trombone", "troop", "trooper", +"trophy", "trouble", "trousers", "trout", "trove", "trowel", "truck", "trumpet", "trunk", "trust", "trustee", "truth", "try", "tsunami", +"tub", "tuba", "tube", "tuber", "tug", "tugboat", "tuition", "tulip", "tumbler", "tummy", "tuna", "tune", "tunic", "tunnel", +"turban", "turf", "turkey", "turmeric", "turn", "turning", "turnip", "turnover", "turnstile", "turret", "turtle", "tusk", "tussle", "tutu", +"tuxedo", "tweet", "tweezers", "twig", "twilight", "twine", "twins", "twist", "twister", "twitter", "type", "typeface", "typewriter", +"typhoon", "ukulele", "ultimatum", "umbrella", "unblinking", "uncertainty", "uncle", "underclothes", "underestimate", "underground", +"underneath", "underpants", "underpass", "undershirt", "understanding", "understatement", "undertaker", "underwear", "underweight", "underwire", +"underwriting", "unemployment", "unibody", "uniform", "uniformity", "union", "unique", "unit", "unity", "universe", "university", "update", +"upgrade", "uplift", "upper", "upstairs", "upward", "urge", "urgency", "urn", "usage", "use", "user", "usher", "usual", "utensil", "utilisation", +"utility", "utilization", "vacation", "vaccine", "vacuum", "vagrant", "valance", "valentine", "validate", "validity", "valley", "valuable", +"value", "vampire", "van", "vanadyl", "vane", "vanilla", "vanity", "variability", "variable", "variant", "variation", "variety", "vascular", +"vase", "vault", "vaulting", "veal", "vector", "vegetable", "vegetarian", "vegetarianism", "vegetation", "vehicle", "veil", "vein", "veldt", +"vellum", "velocity", "velodrome", "velvet", "vendor", "veneer", "vengeance", "venison", "venom", "venti", "venture", "venue", "veranda", +"verb", "verdict", "verification", "vermicelli", "vernacular", "verse", "version", "vertigo", "verve", "vessel", "vest", "vestment", +"vet", "veteran", "veterinarian", "veto", "viability", "vibe", "vibraphone", "vibration", "vibrissae", "vice", "vicinity", "victim", +"victory", "video", "view", "viewer", "vignette", "villa", "village", "vine", "vinegar", "vineyard", "vintage", "vintner", "vinyl", "viola", +"violation", "violence", "violet", "violin", "virginal", "virtue", "virus", "visa", "viscose", "vise", "vision", "visit", "visitor", +"visor", "vista", "visual", "vitality", "vitamin", "vitro", "vivo", "vixen", "vodka", "vogue", "voice", "void", "vol", "volatility", +"volcano", "volleyball", "volume", "volunteer", "volunteering", "vomit", "vote", "voter", "voting", "voyage", "vulture", "wad", "wafer", +"waffle", "wage", "wagon", "waist", "waistband", "wait", "waiter", "waiting", "waitress", "waiver", "wake", "walk", "walker", "walking", +"walkway", "wall", "wallaby", "wallet", "walnut", "walrus", "wampum", "wannabe", "want", "war", "warden", "wardrobe", "warfare", "warlock", +"warlord", "warming", "warmth", "warning", "warrant", "warren", "warrior", "wasabi", "wash", "washbasin", "washcloth", "washer", +"washtub", "wasp", "waste", "wastebasket", "wasting", "watch", "watcher", "watchmaker", "water", "waterbed", "watercress", "waterfall", +"waterfront", "watermelon", "waterskiing", "waterspout", "waterwheel", "wave", "waveform", "wax", "way", "weakness", "wealth", "weapon", +"wear", "weasel", "weather", "web", "webinar", "webmail", "webpage", "website", "wedding", "wedge", "weed", "weeder", "weedkiller", "week", +"weekend", "weekender", "weight", "weird", "welcome", "welfare", "well", "west", "western", "wetland", "wetsuit", +"whack", "whale", "wharf", "wheat", "wheel", "whelp", "whey", "whip", "whirlpool", "whirlwind", "whisker", "whiskey", "whisper", "whistle", +"white", "whole", "wholesale", "wholesaler", "whorl", "wick", "widget", "widow", "width", "wife", "wifi", "wild", "wildebeest", "wilderness", +"wildlife", "will", "willingness", "willow", "win", "wind", "windage", "window", "windscreen", "windshield", "wine", "winery", +"wing", "wingman", "wingtip", "wink", "winner", "winter", "wire", "wiretap", "wiring", "wisdom", "wiseguy", "wish", "wisteria", "wit", +"witch", "withdrawal", "witness", "wok", "wolf", "woman", "wombat", "wonder", "wont", "wood", "woodchuck", "woodland", +"woodshed", "woodwind", "wool", "woolens", "word", "wording", "work", "workbench", "worker", "workforce", "workhorse", "working", "workout", +"workplace", "workshop", "world", "worm", "worry", "worship", "worshiper", "worth", "wound", "wrap", "wraparound", "wrapper", "wrapping", +"wreck", "wrecker", "wren", "wrench", "wrestler", "wriggler", "wrinkle", "wrist", "writer", "writing", "wrong", "xylophone", "yacht", +"yahoo", "yak", "yam", "yang", "yard", "yarmulke", "yarn", "yawl", "year", "yeast", "yellow", "yellowjacket", "yesterday", "yew", "yin", +"yoga", "yogurt", "yoke", "yolk", "young", "youngster", "yourself", "youth", "yoyo", "yurt", "zampone", "zebra", "zebrafish", "zen", +"zephyr", "zero", "ziggurat", "zinc", "zipper", "zither", "zombie", "zone", "zoo", "zoologist", "zoology", "zucchini" +}; + + +std::string_view obfuscateWord(std::string_view src, WordMap & obfuscate_map, WordSet & used_nouns, SipHash hash_func) +{ + /// Prevent using too many nouns + if (obfuscate_map.size() * 2 > nouns.size()) + throw Exception("Too many unique identifiers in queries", ErrorCodes::TOO_MANY_TEMPORARY_COLUMNS); + + std::string_view & mapped = obfuscate_map[src]; + if (!mapped.empty()) + return mapped; + + hash_func.update(src.data(), src.size()); + std::string_view noun = nouns.begin()[hash_func.get64() % nouns.size()]; + + /// Prevent collisions + while (!used_nouns.insert(noun).second) + { + hash_func.update('\0'); + noun = nouns.begin()[hash_func.get64() % nouns.size()]; + } + + mapped = noun; + return mapped; +} + + +void obfuscateIdentifier(std::string_view src, WriteBuffer & result, WordMap & obfuscate_map, WordSet & used_nouns, SipHash hash_func) +{ + /// Find words in form 'snake_case', 'CamelCase' or 'ALL_CAPS'. + + const char * src_pos = src.data(); + const char * src_end = src_pos + src.size(); + + const char * word_begin = src_pos; + bool word_has_alphanumerics = false; + + auto append_word = [&] + { + std::string_view word(word_begin, src_pos - word_begin); + + if (keep_words.count(word)) + { + result.write(word.data(), word.size()); + } + else + { + std::string_view obfuscated_word = obfuscateWord(word, obfuscate_map, used_nouns, hash_func); + + /// Match the style of source word. + bool first_caps = !word.empty() && isUpperAlphaASCII(word[0]); + bool all_caps = first_caps && word.size() >= 2 && isUpperAlphaASCII(word[1]); + + for (size_t i = 0, size = obfuscated_word.size(); i < size; ++i) + { + if (all_caps || (i == 0 && first_caps)) + result.write(toUpperIfAlphaASCII(obfuscated_word[i])); + else + result.write(obfuscated_word[i]); + } + } + + word_begin = src_pos; + word_has_alphanumerics = false; + }; + + while (src_pos < src_end) + { + if (isAlphaNumericASCII(src_pos[0])) + word_has_alphanumerics = true; + + if (word_has_alphanumerics && src_pos[0] == '_') + { + append_word(); + result.write('_'); + ++word_begin; + } + else if (word_has_alphanumerics && isUpperAlphaASCII(src_pos[0]) && isLowerAlphaASCII(src_pos[-1])) /// xX + { + append_word(); + } + + ++src_pos; + } + + if (word_begin < src_pos) + append_word(); +} + + +void obfuscateLiteral(std::string_view src, WriteBuffer & result, SipHash hash_func) +{ + const char * src_pos = src.data(); + const char * src_end = src_pos + src.size(); + + while (src_pos < src_end) + { + /// Date + if (src_pos + strlen("0000-00-00") <= src_end + && isNumericASCII(src_pos[0]) + && isNumericASCII(src_pos[1]) + && isNumericASCII(src_pos[2]) + && isNumericASCII(src_pos[3]) + && src_pos[4] == '-' + && isNumericASCII(src_pos[5]) + && isNumericASCII(src_pos[6]) + && src_pos[7] == '-' + && isNumericASCII(src_pos[8]) + && isNumericASCII(src_pos[9])) + { + DayNum date; + ReadBufferFromMemory in(src_pos, strlen("0000-00-00")); + readDateText(date, in); + + SipHash hash_func_date = hash_func; + + if (date != 0) + { + date += hash_func_date.get64() % 256; + } + + writeDateText(date, result); + src_pos += strlen("0000-00-00"); + + /// DateTime + if (src_pos + strlen(" 00:00:00") <= src_end + && isNumericASCII(src_pos[1]) + && isNumericASCII(src_pos[2]) + && src_pos[3] == ':' + && isNumericASCII(src_pos[4]) + && isNumericASCII(src_pos[5]) + && src_pos[6] == ':' + && isNumericASCII(src_pos[7]) + && isNumericASCII(src_pos[8])) + { + result.write(src_pos[0]); + + hash_func_date.update(src_pos + 1, strlen("00:00:00")); + + uint64_t hash_value = hash_func_date.get64(); + uint32_t new_hour = hash_value % 24; + hash_value /= 24; + uint32_t new_minute = hash_value % 60; + hash_value /= 60; + uint32_t new_second = hash_value % 60; + + result.write('0' + (new_hour / 10)); + result.write('0' + (new_hour % 10)); + result.write(':'); + result.write('0' + (new_minute / 10)); + result.write('0' + (new_minute % 10)); + result.write(':'); + result.write('0' + (new_second / 10)); + result.write('0' + (new_second % 10)); + + src_pos += strlen(" 00:00:00"); + } + } + else if (isNumericASCII(src_pos[0])) + { + /// Number + if (src_pos[0] == '0' || src_pos[0] == '1') + { + /// Keep zero and one as is. + result.write(src_pos[0]); + ++src_pos; + } + else + { + ReadBufferFromMemory in(src_pos, src_end - src_pos); + uint64_t num; + readIntText(num, in); + SipHash hash_func_num = hash_func; + hash_func_num.update(src_pos, in.count()); + src_pos += in.count(); + + /// Obfuscate number but keep it within same power of two range. + + uint64_t obfuscated = hash_func_num.get64(); + uint64_t log2 = bitScanReverse(num); + + obfuscated = (1ULL << log2) + obfuscated % (1ULL << log2); + writeIntText(obfuscated, result); + } + } + else if (src_pos + 1 < src_end + && (src_pos[0] == 'e' || src_pos[0] == 'E') + && (isNumericASCII(src_pos[1]) || (src_pos[1] == '-' && src_pos + 2 < src_end && isNumericASCII(src_pos[2])))) + { + /// Something like an exponent of floating point number. Keep it as is. + /// But if it looks like a large number, overflow it into 16 bit. + + result.write(src_pos[0]); + ++src_pos; + + ReadBufferFromMemory in(src_pos, src_end - src_pos); + int16_t num; + readIntText(num, in); + writeIntText(num, result); + src_pos += in.count(); + } + else if (isAlphaASCII(src_pos[0])) + { + /// Alphabetial characters + + const char * alpha_end = src_pos + 1; + while (alpha_end < src_end && isAlphaASCII(*alpha_end)) + ++alpha_end; + + hash_func.update(src_pos, alpha_end - src_pos); + pcg64 rng(hash_func.get64()); + + while (src_pos < alpha_end) + { + auto random = rng(); + if (isLowerAlphaASCII(*src_pos)) + result.write('a' + random % 26); + else + result.write('A' + random % 26); + + ++src_pos; + } + } + else if (isASCII(src_pos[0])) + { + /// Punctuation, whitespace and control characters - keep as is. + + result.write(src_pos[0]); + ++src_pos; + } + else if (src_pos[0] <= '\xBF') + { + /// Continuation of UTF-8 sequence. + hash_func.update(src_pos[0]); + uint64_t hash = hash_func.get64(); + + char c = 0x80 + hash % (0xC0 - 0x80); + result.write(c); + + ++src_pos; + } + else if (src_pos[0]) + { + /// Start of UTF-8 sequence. + hash_func.update(src_pos[0]); + uint64_t hash = hash_func.get64(); + + if (src_pos[0] < '\xE0') + { + char c = 0xC0 + hash % 32; + result.write(c); + } + else if (src_pos[0] < '\xF0') + { + char c = 0xE0 + hash % 16; + result.write(c); + } + else + { + char c = 0xF0 + hash % 8; + result.write(c); + } + + ++src_pos; + } + } +} + +} + + +void obfuscateQueries( + std::string_view src, + WriteBuffer & result, + WordMap & obfuscate_map, + WordSet & used_nouns, + SipHash hash_func, + KnownIdentifierFunc known_identifier_func) +{ + Lexer lexer(src.data(), src.data() + src.size()); + while (true) + { + Token token = lexer.nextToken(); + std::string_view whole_token(token.begin, token.size()); + + if (token.isEnd()) + break; + + if (token.type == TokenType::BareWord) + { + if (keywords.count(whole_token) + || known_identifier_func(whole_token)) + { + /// Keep keywords as is. + result.write(token.begin, token.size()); + } + else + { + /// Obfuscate identifiers + obfuscateIdentifier(whole_token, result, obfuscate_map, used_nouns, hash_func); + } + } + else if (token.type == TokenType::QuotedIdentifier) + { + assert(token.size() >= 2); + + /// Write quotes and the obfuscated content inside. + result.write(*token.begin); + obfuscateIdentifier({token.begin + 1, token.size() - 2}, result, obfuscate_map, used_nouns, hash_func); + result.write(token.end[-1]); + } + else if (token.type == TokenType::Number) + { + obfuscateLiteral(whole_token, result, hash_func); + } + else if (token.type == TokenType::StringLiteral) + { + assert(token.size() >= 2); + + result.write(*token.begin); + obfuscateLiteral({token.begin + 1, token.size() - 2}, result, hash_func); + result.write(token.end[-1]); + } + else if (token.type == TokenType::Comment) + { + /// Skip comments - they may contain confidential info. + } + else + { + /// Everyting else is kept as is. + result.write(token.begin, token.size()); + } + } +} + +} + diff --git a/src/Parsers/obfuscateQueries.h b/src/Parsers/obfuscateQueries.h new file mode 100644 index 00000000000..214237e19cf --- /dev/null +++ b/src/Parsers/obfuscateQueries.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +class WriteBuffer; + +using WordMap = std::unordered_map; +using WordSet = std::unordered_set; +using KnownIdentifierFunc = std::function; + +void obfuscateQueries( + std::string_view src, + WriteBuffer & result, + WordMap & obfuscate_map, + WordSet & used_nouns, + SipHash hash_func, + KnownIdentifierFunc known_identifier_func); + +} diff --git a/src/Parsers/tests/gtest_obfuscate_queries.cpp b/src/Parsers/tests/gtest_obfuscate_queries.cpp new file mode 100644 index 00000000000..ada15109d17 --- /dev/null +++ b/src/Parsers/tests/gtest_obfuscate_queries.cpp @@ -0,0 +1,92 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +using namespace DB; + + +TEST(ObfuscateQueries, Test1) +{ + WordMap obfuscated_words_map; + WordSet used_nouns; + SipHash hash_func; + + std::string salt = "Hello, world"; + hash_func.update(salt); + + SharedContextHolder shared_context; + const ContextHolder & context_holder = getContext(); + + registerFunctions(); + registerAggregateFunctions(); + registerTableFunctions(); + registerStorages(); + + std::unordered_set additional_names; + + auto all_known_storage_names = StorageFactory::instance().getAllRegisteredNames(); + auto all_known_data_type_names = DataTypeFactory::instance().getAllRegisteredNames(); + + additional_names.insert(all_known_storage_names.begin(), all_known_storage_names.end()); + additional_names.insert(all_known_data_type_names.begin(), all_known_data_type_names.end()); + + KnownIdentifierFunc is_known_identifier = [&](std::string_view name) + { + std::string what(name); + + return FunctionFactory::instance().tryGet(what, context_holder.context) != nullptr + || AggregateFunctionFactory::instance().isAggregateFunctionName(what) + || TableFunctionFactory::instance().isTableFunctionName(what) + || additional_names.count(what); + }; + + WriteBufferFromOwnString out; + + obfuscateQueries( + R"( +SELECT + VisitID, + Goals.ID, Goals.EventTime, + WatchIDs, + EAction.ProductName, EAction.ProductPrice, EAction.ProductCurrency, EAction.ProductQuantity, EAction.EventTime, EAction.Type +FROM merge.visits_v2 +WHERE + StartDate >= '2020-09-17' AND StartDate <= '2020-09-25' + AND CounterID = 24226447 + AND intHash32(UserID) = 416638616 AND intHash64(UserID) = 13269091395366875299 + AND VisitID IN (5653048135597886819, 5556254872710352304, 5516214175671455313, 5476714937521999313, 5464051549483503043) + AND Sign = 1 +)", + out, obfuscated_words_map, used_nouns, hash_func, is_known_identifier); + + EXPECT_EQ(out.str(), R"( +SELECT + CorduroyID, + Steel.ID, Steel.AcornSidestream, + WealthBRANCH, + GOVERNMENT.SedimentName, GOVERNMENT.SedimentExhaustion, GOVERNMENT.SedimentFencing, GOVERNMENT.SedimentOpossum, GOVERNMENT.AcornSidestream, GOVERNMENT.Lute +FROM merge.luncheonette_pants +WHERE + GovernanceCreche >= '2021-04-16' AND GovernanceCreche <= '2021-04-24' + AND StarboardID = 26446940 + AND intHash32(MessyID) = 474525514 AND intHash64(MessyID) = 13916317227779800149 + AND CorduroyID IN (5223158832904664474, 5605365157729463108, 7543250143731591192, 8715842063486405567, 7837015536326316923) + AND Tea = 1 +)"); +} + From ed4ff51ed8328c087575d4c3c4bb703356052290 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:38:59 +0300 Subject: [PATCH 376/390] Integrate into clickhouse-format; add test and comments --- programs/format/Format.cpp | 98 ++++++++++++++++--- src/Parsers/obfuscateQueries.cpp | 2 +- src/Parsers/obfuscateQueries.h | 21 ++++ .../01508_query_obfuscator.reference | 1 + .../0_stateless/01508_query_obfuscator.sh | 6 ++ 5 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/01508_query_obfuscator.reference create mode 100755 tests/queries/0_stateless/01508_query_obfuscator.sh diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index daf2d671568..01f952bf95e 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -1,13 +1,29 @@ #include +#include +#include #include #include #include +#include #include #include #include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wmissing-declarations" @@ -22,6 +38,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ("oneline", "format in single line") ("quiet,q", "just check syntax, no output on success") ("multiquery,n", "allow multiple queries in the same file") + ("obfuscate", "obfuscate instead of formatting") + ("seed", po::value(), "seed (arbitrary string) that determines the result of obfuscation") ; boost::program_options::variables_map options; @@ -40,10 +58,17 @@ int mainEntryClickHouseFormat(int argc, char ** argv) bool oneline = options.count("oneline"); bool quiet = options.count("quiet"); bool multiple = options.count("multiquery"); + bool obfuscate = options.count("obfuscate"); - if (quiet && (hilite || oneline)) + if (quiet && (hilite || oneline || obfuscate)) { - std::cerr << "Options 'hilite' or 'oneline' have no sense in 'quiet' mode." << std::endl; + std::cerr << "Options 'hilite' or 'oneline' or 'obfuscate' have no sense in 'quiet' mode." << std::endl; + return 2; + } + + if (obfuscate && (hilite || oneline || quiet)) + { + std::cerr << "Options 'hilite' or 'oneline' or 'quiet' have no sense in 'obfuscate' mode." << std::endl; return 2; } @@ -51,21 +76,66 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ReadBufferFromFileDescriptor in(STDIN_FILENO); readStringUntilEOF(query, in); - const char * pos = query.data(); - const char * end = pos + query.size(); - - ParserQuery parser(end); - do + if (obfuscate) { - ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - if (!quiet) + WordMap obfuscated_words_map; + WordSet used_nouns; + SipHash hash_func; + + if (options.count("seed")) { - formatAST(*res, std::cout, hilite, oneline); - if (multiple) - std::cout << "\n;\n"; - std::cout << std::endl; + std::string seed; + hash_func.update(options["seed"].as()); } - } while (multiple && pos != end); + + SharedContextHolder shared_context = Context::createShared(); + Context context = Context::createGlobal(shared_context.get()); + context.makeGlobalContext(); + + registerFunctions(); + registerAggregateFunctions(); + registerTableFunctions(); + registerStorages(); + + std::unordered_set additional_names; + + auto all_known_storage_names = StorageFactory::instance().getAllRegisteredNames(); + auto all_known_data_type_names = DataTypeFactory::instance().getAllRegisteredNames(); + + additional_names.insert(all_known_storage_names.begin(), all_known_storage_names.end()); + additional_names.insert(all_known_data_type_names.begin(), all_known_data_type_names.end()); + + KnownIdentifierFunc is_known_identifier = [&](std::string_view name) + { + std::string what(name); + + return FunctionFactory::instance().tryGet(what, context) != nullptr + || AggregateFunctionFactory::instance().isAggregateFunctionName(what) + || TableFunctionFactory::instance().isTableFunctionName(what) + || additional_names.count(what); + }; + + WriteBufferFromFileDescriptor out(STDOUT_FILENO); + obfuscateQueries(query, out, obfuscated_words_map, used_nouns, hash_func, is_known_identifier); + } + else + { + const char * pos = query.data(); + const char * end = pos + query.size(); + + ParserQuery parser(end); + do + { + ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + if (!quiet) + { + formatAST(*res, std::cout, hilite, oneline); + if (multiple) + std::cout << "\n;\n"; + std::cout << std::endl; + } + } while (multiple && pos != end); + } } catch (...) { diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 66221005d77..9dcf3d6ada3 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -836,7 +836,7 @@ void obfuscateLiteral(std::string_view src, WriteBuffer & result, SipHash hash_f ++src_pos; } - else if (src_pos[0]) + else { /// Start of UTF-8 sequence. hash_func.update(src_pos[0]); diff --git a/src/Parsers/obfuscateQueries.h b/src/Parsers/obfuscateQueries.h index 214237e19cf..2e65ec427f6 100644 --- a/src/Parsers/obfuscateQueries.h +++ b/src/Parsers/obfuscateQueries.h @@ -18,6 +18,27 @@ using WordMap = std::unordered_map; using WordSet = std::unordered_set; using KnownIdentifierFunc = std::function; +/** Takes one or multiple queries and obfuscate them by replacing identifiers to pseudorandom words + * and replacing literals to random values, while preserving the structure of the queries and the general sense. + * + * Its intended use case is when the user wants to share their queries for testing and debugging + * but is afraid to disclose the details about their column names, domain area and values of constants. + * + * It can obfuscate multiple queries in consistent fashion - identical names will be transformed to identical results. + * + * The function is not guaranteed to always give correct result or to be secure. It's implemented in "best effort" fashion. + * + * @param src - a string with source queries. + * @param result - where the obfuscated queries will be written. + * @param obfuscate_map - information about substituted identifiers + * (pass empty map at the beginning or reuse it from previous invocation to get consistent result) + * @param used_nouns - information about words used for substitution + * (pass empty set at the beginning or reuse it from previous invocation to get consistent result) + * @param hash_func - hash function that will be used as a pseudorandom source, + * it's recommended to preseed the function before passing here. + * @param known_identifier_func - a function that returns true if identifier is known name + * (of function, aggregate function, etc. that should be kept as is). If it returns false, identifier will be obfuscated. + */ void obfuscateQueries( std::string_view src, WriteBuffer & result, diff --git a/tests/queries/0_stateless/01508_query_obfuscator.reference b/tests/queries/0_stateless/01508_query_obfuscator.reference new file mode 100644 index 00000000000..e2f61cf345d --- /dev/null +++ b/tests/queries/0_stateless/01508_query_obfuscator.reference @@ -0,0 +1 @@ +SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Meantime_id_brisketHeavy, exclamation(Grit), avgIf(remote('128.0.0.1')) diff --git a/tests/queries/0_stateless/01508_query_obfuscator.sh b/tests/queries/0_stateless/01508_query_obfuscator.sh new file mode 100755 index 00000000000..797271edb30 --- /dev/null +++ b/tests/queries/0_stateless/01508_query_obfuscator.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT 123, 'Test://2020-01-01hello1234 at 2000-01-01T01:02:03', 12e100, Gibberish_id_testCool, hello(World), avgIf(remote('127.0.0.1'))" From 673f72cf8c147fa0252052e87ae0b2346d2e0e6f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:48:09 +0300 Subject: [PATCH 377/390] Remove some profanity --- src/Parsers/obfuscateQueries.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 9dcf3d6ada3..e8ea68ba0c6 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -76,7 +76,7 @@ std::initializer_list nouns "architect", "architecture", "archives", "area", "arena", "argument", "arithmetic", "ark", "arm", "armadillo", "armament", "armchair", "armoire", "armor", "armour", "armpit", "armrest", "army", "arrangement", "array", "arrest", "arrival", "arrogance", "arrow", "art", "artery", "arthur", "artichoke", "article", "artifact", "artificer", "artist", "ascend", "ascent", "ascot", "ash", "ashram", "ashtray", -"aside", "asparagus", "aspect", "asphalt", "aspic", "ass", "assassination", "assault", "assembly", "assertion", "assessment", "asset", +"aside", "asparagus", "aspect", "asphalt", "aspic", "assassination", "assault", "assembly", "assertion", "assessment", "asset", "assignment", "assist", "assistance", "assistant", "associate", "association", "assumption", "assurance", "asterisk", "astrakhan", "astrolabe", "astrologer", "astrology", "astronomy", "asymmetry", "atelier", "atheist", "athlete", "athletics", "atmosphere", "atom", "atrium", "attachment", "attack", "attacker", "attainment", "attempt", "attendance", "attendant", "attention", "attenuation", "attic", "attitude", "attorney", @@ -182,7 +182,7 @@ std::initializer_list nouns "current", "curriculum", "curry", "curse", "cursor", "curtailment", "curtain", "curve", "cushion", "custard", "custody", "custom", "customer", "cut", "cuticle", "cutlet", "cutover", "cutting", "cyclamen", "cycle", "cyclone", "cyclooxygenase", "cygnet", "cylinder", "cymbal", "cynic", "cyst", "cytokine", "cytoplasm", "dad", "daddy", "daffodil", "dagger", "dahlia", "daikon", "daily", "dairy", "daisy", "dam", "damage", -"dame", "damn", "dance", "dancer", "dancing", "dandelion", "danger", "dare", "dark", "darkness", "darn", "dart", "dash", "dashboard", +"dame", "dance", "dancer", "dancing", "dandelion", "danger", "dare", "dark", "darkness", "darn", "dart", "dash", "dashboard", "data", "database", "date", "daughter", "dawn", "day", "daybed", "daylight", "dead", "deadline", "deal", "dealer", "dealing", "dearest", "death", "deathwatch", "debate", "debris", "debt", "debtor", "decade", "decadence", "decency", "decimal", "decision", "decision-making", "deck", "declaration", "declination", "decline", "decoder", "decongestant", "decoration", "decrease", "decryption", "dedication", "deduce", @@ -193,8 +193,7 @@ std::initializer_list nouns "description", "desert", "design", "designation", "designer", "desire", "desk", "desktop", "dessert", "destination", "destiny", "destroyer", "destruction", "detail", "detainee", "detainment", "detection", "detective", "detector", "detention", "determination", "detour", "devastation", "developer", "developing", "development", "developmental", "deviance", "deviation", "device", "devil", "dew", "dhow", "diabetes", "diadem", -"diagnosis", "diagram", "dial", "dialect", "dialogue", "diam", "diamond", "diaper", "diaphragm", "diarist", "diary", "dibble", "dick", -"dickey", "dictaphone", "dictator", "diction", "dictionary", "die", "diesel", "diet", "difference", "differential", "difficulty", "diffuse", +"diagnosis", "diagram", "dial", "dialect", "dialogue", "diam", "diamond", "diaper", "diaphragm", "diarist", "diary", "dibble", "dickey", "dictaphone", "dictator", "diction", "dictionary", "die", "diesel", "diet", "difference", "differential", "difficulty", "diffuse", "dig", "digestion", "digestive", "digger", "digging", "digit", "dignity", "dilapidation", "dill", "dilution", "dime", "dimension", "dimple", "diner", "dinghy", "dining", "dinner", "dinosaur", "dioxide", "dip", "diploma", "diplomacy", "dipstick", "direction", "directive", "director", "directory", "dirndl", "dirt", "disability", "disadvantage", "disagreement", "disappointment", "disarmament", "disaster", "discharge", @@ -254,7 +253,7 @@ std::initializer_list nouns "fortress", "fortune", "forum", "foundation", "founder", "founding", "fountain", "fourths", "fowl", "fox", "foxglove", "fraction", "fragrance", "frame", "framework", "fratricide", "fraud", "fraudster", "freak", "freckle", "freedom", "freelance", "freezer", "freezing", "freight", "freighter", "frenzy", "freon", "frequency", "fresco", "friction", "fridge", "friend", "friendship", "fries", "frigate", "fright", "fringe", -"fritter", "frock", "frog", "front", "frontier", "frost", "frosting", "frown", "fruit", "frustration", "fry", "fuck", "fuel", "fugato", +"fritter", "frock", "frog", "front", "frontier", "frost", "frosting", "frown", "fruit", "frustration", "fry", "fuel", "fugato", "fulfillment", "full", "fun", "function", "functionality", "fund", "funding", "fundraising", "funeral", "fur", "furnace", "furniture", "furry", "fusarium", "futon", "future", "gadget", "gaffe", "gaffer", "gain", "gaiters", "gale", "gallery", "galley", "gallon", "galoshes", "gambling", "game", "gamebird", "gaming", "gander", "gang", "gap", "garage", "garb", "garbage", "garden", @@ -413,7 +412,7 @@ std::initializer_list nouns "picture", "picturesque", "pie", "piece", "pier", "piety", "pig", "pigeon", "piglet", "pigpen", "pigsty", "pike", "pilaf", "pile", "pilgrim", "pilgrimage", "pill", "pillar", "pillbox", "pillow", "pilot", "pimp", "pimple", "pin", "pinafore", "pine", "pineapple", "pinecone", "ping", "pink", "pinkie", "pinot", "pinstripe", "pint", "pinto", "pinworm", "pioneer", "pipe", "pipeline", "piracy", "pirate", -"piss", "pistol", "pit", "pita", "pitch", "pitcher", "pitching", "pith", "pizza", "place", "placebo", "placement", "placode", "plagiarism", +"pistol", "pit", "pita", "pitch", "pitcher", "pitching", "pith", "pizza", "place", "placebo", "placement", "placode", "plagiarism", "plain", "plaintiff", "plan", "plane", "planet", "planning", "plant", "plantation", "planter", "planula", "plaster", "plasterboard", "plastic", "plate", "platelet", "platform", "platinum", "platter", "platypus", "play", "player", "playground", "playroom", "playwright", "plea", "pleasure", "pleat", "pledge", "plenty", "plier", "pliers", "plight", "plot", "plough", "plover", "plow", "plowman", "plug", @@ -466,7 +465,7 @@ std::initializer_list nouns "riot", "rip", "ripple", "rise", "riser", "risk", "rite", "ritual", "river", "riverbed", "rivulet", "road", "roadway", "roar", "roast", "robe", "robin", "robot", "robotics", "rock", "rocker", "rocket", "rod", "role", "roll", "roller", "romaine", "romance", "roof", "room", "roommate", "rooster", "root", "rope", "rose", "rosemary", "roster", "rostrum", "rotation", "round", "roundabout", "route", -"router", "routine", "row", "rowboat", "rowing", "rubber", "rubbish", "rubric", "ruby", "ruckus", "rudiment", "ruffle", "rug", "rugby", +"router", "routine", "row", "rowboat", "rowing", "rubber", "rubric", "ruby", "ruckus", "rudiment", "ruffle", "rug", "rugby", "ruin", "rule", "ruler", "ruling", "rum", "rumor", "run", "runaway", "runner", "running", "runway", "rush", "rust", "rutabaga", "rye", "sabre", "sac", "sack", "saddle", "sadness", "safari", "safe", "safeguard", "safety", "saffron", "sage", "sail", "sailboat", "sailing", "sailor", "saint", "sake", "salad", "salami", "salary", "sale", "salesman", "salmon", "salon", "saloon", "salsa", "salt", "salute", "samovar", @@ -486,7 +485,7 @@ std::initializer_list nouns "setback", "setting", "settlement", "settler", "severity", "sewer", "sex", "sexuality", "shack", "shackle", "shade", "shadow", "shadowbox", "shakedown", "shaker", "shallot", "shallows", "shame", "shampoo", "shanty", "shape", "share", "shareholder", "shark", "shaw", "shawl", "shear", "shearling", "sheath", "shed", "sheep", "sheet", "shelf", "shell", "shelter", "sherbet", "sherry", "shield", "shift", "shin", -"shine", "shingle", "ship", "shipper", "shipping", "shipyard", "shirt", "shirtdress", "shit", "shoat", "shock", "shoe", "shoe-horn", +"shine", "shingle", "ship", "shipper", "shipping", "shipyard", "shirt", "shirtdress", "shoat", "shock", "shoe", "shoe-horn", "shoehorn", "shoelace", "shoemaker", "shoes", "shoestring", "shofar", "shoot", "shootdown", "shop", "shopper", "shopping", "shore", "shoreline", "short", "shortage", "shorts", "shortwave", "shot", "shoulder", "shout", "shovel", "show", "shower", "shred", "shrimp", "shrine", "shutdown", "sibling", "sick", "sickness", "side", "sideboard", "sideburns", "sidecar", "sidestream", "sidewalk", "siding", @@ -494,7 +493,7 @@ std::initializer_list nouns "silicon", "silk", "silkworm", "sill", "silly", "silo", "silver", "similarity", "simple", "simplicity", "simplification", "simvastatin", "sin", "singer", "singing", "singular", "sink", "sinuosity", "sip", "sir", "sister", "sitar", "site", "situation", "size", "skate", "skating", "skean", "skeleton", "ski", "skiing", "skill", "skin", "skirt", "skull", "skullcap", "skullduggery", "skunk", "sky", -"skylight", "skyline", "skyscraper", "skywalk", "slang", "slapstick", "slash", "slate", "slave", "slavery", "slaw", "sled", "sledge", +"skylight", "skyline", "skyscraper", "skywalk", "slang", "slapstick", "slash", "slate", "slavery", "slaw", "sled", "sledge", "sleep", "sleepiness", "sleeping", "sleet", "sleuth", "slice", "slide", "slider", "slime", "slip", "slipper", "slippers", "slope", "slot", "sloth", "slump", "smell", "smelting", "smile", "smith", "smock", "smog", "smoke", "smoking", "smolt", "smuggling", "snack", "snail", "snake", "snakebite", "snap", "snarl", "sneaker", "sneakers", "sneeze", "sniffle", "snob", "snorer", "snow", "snowboarding", "snowflake", From 22861b0c3dcd864584954b79436000a38306d130 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:49:30 +0300 Subject: [PATCH 378/390] Remove some profanity --- src/Parsers/obfuscateQueries.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index e8ea68ba0c6..03f0cef2605 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -171,7 +171,7 @@ std::initializer_list nouns "councilperson", "counsel", "counseling", "counselling", "counsellor", "counselor", "count", "counter", "counterpart", "counterterrorism", "countess", "country", "countryside", "county", "couple", "coupon", "courage", "course", "court", "courthouse", "courtroom", "cousin", "covariate", "cover", "coverage", "coverall", "cow", "cowbell", "cowboy", "coyote", "crab", "crack", "cracker", "crackers", -"cradle", "craft", "craftsman", "cranberry", "crane", "cranky", "crap", "crash", "crate", "cravat", "craw", "crawdad", "crayfish", "crayon", +"cradle", "craft", "craftsman", "cranberry", "crane", "cranky", "crash", "crate", "cravat", "craw", "crawdad", "crayfish", "crayon", "crazy", "cream", "creation", "creationism", "creationist", "creative", "creativity", "creator", "creature", "creche", "credential", "credenza", "credibility", "credit", "creditor", "creek", "creme brulee", "crepe", "crest", "crew", "crewman", "crewmate", "crewmember", "crewmen", "cria", "crib", "cribbage", "cricket", "cricketer", "crime", "criminal", "crinoline", "crisis", "crisp", "criteria", "criterion", From d78fc63deb0d8a37e7a10e045b5807e9dd154ca7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:50:00 +0300 Subject: [PATCH 379/390] Remove duplicate word --- src/Parsers/obfuscateQueries.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 03f0cef2605..17e751f5149 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -450,7 +450,7 @@ std::initializer_list nouns "recipient", "reciprocity", "reclamation", "recliner", "recognition", "recollection", "recommendation", "reconsideration", "record", "recorder", "recording", "recovery", "recreation", "recruit", "rectangle", "red", "redesign", "redhead", "redirect", "rediscovery", "reduction", "reef", "refectory", "reference", "referendum", "reflection", "reform", "refreshments", "refrigerator", "refuge", "refund", "refusal", -"refuse", "regard", "regime", "region", "regionalism", "register", "registration", "registry", "regret", "regulation", "regulator", "rehospitalisation", +"refuse", "regard", "regime", "region", "regionalism", "register", "registration", "registry", "regret", "regulation", "regulator", "rehospitalization", "reindeer", "reinscription", "reject", "relation", "relationship", "relative", "relaxation", "relay", "release", "reliability", "relief", "religion", "relish", "reluctance", "remains", "remark", "reminder", "remnant", "remote", "removal", "renaissance", "rent", "reorganisation", "reorganization", "repair", "reparation", "repayment", "repeat", "replacement", "replica", "replication", "reply", From fc10803deffd4585bb516753552a8c1eb387f246 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:50:56 +0300 Subject: [PATCH 380/390] Fix comment --- src/Parsers/obfuscateQueries.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/obfuscateQueries.h b/src/Parsers/obfuscateQueries.h index 2e65ec427f6..0a192649a92 100644 --- a/src/Parsers/obfuscateQueries.h +++ b/src/Parsers/obfuscateQueries.h @@ -18,7 +18,7 @@ using WordMap = std::unordered_map; using WordSet = std::unordered_set; using KnownIdentifierFunc = std::function; -/** Takes one or multiple queries and obfuscate them by replacing identifiers to pseudorandom words +/** Takes one or multiple queries and obfuscates them by replacing identifiers to pseudorandom words * and replacing literals to random values, while preserving the structure of the queries and the general sense. * * Its intended use case is when the user wants to share their queries for testing and debugging From bda2c2f58fcc590bed044200f4a5e0c563691d54 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 03:53:26 +0300 Subject: [PATCH 381/390] Remove sensitive words --- src/Parsers/obfuscateQueries.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 17e751f5149..0d62fe435d8 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -283,10 +283,10 @@ std::initializer_list nouns "hearsay", "heart", "heartache", "heartbeat", "hearth", "hearthside", "heartwood", "heat", "heater", "heating", "heaven", "heavy", "hectare", "hedge", "hedgehog", "heel", "heifer", "height", "heir", "heirloom", "helicopter", "helium", "hell", "hellcat", "hello", "helmet", "helo", "help", "hemisphere", "hemp", "hen", "hepatitis", "herb", "herbs", "heritage", "hermit", "hero", "heroine", "heron", -"herring", "hesitation", "heterosexual", "hexagon", "heyday", "hiccups", "hide", "hierarchy", "high", "highland", "highlight", +"herring", "hesitation", "hexagon", "heyday", "hiccups", "hide", "hierarchy", "high", "highland", "highlight", "highway", "hike", "hiking", "hill", "hint", "hip", "hippodrome", "hippopotamus", "hire", "hiring", "historian", "history", "hit", "hive", "hobbit", "hobby", "hockey", "hoe", "hog", "hold", "holder", "hole", "holiday", "home", "homeland", "homeownership", "hometown", "homework", -"homicide", "homogenate", "homonym", "homosexual", "homosexuality", "honesty", "honey", "honeybee", "honeydew", "honor", "honoree", "hood", +"homicide", "homogenate", "homonym", "honesty", "honey", "honeybee", "honeydew", "honor", "honoree", "hood", "hoof", "hook", "hop", "hope", "hops", "horde", "horizon", "hormone", "horn", "hornet", "horror", "horse", "horseradish", "horst", "hose", "hosiery", "hospice", "hospital", "hospitalisation", "hospitality", "hospitalization", "host", "hostel", "hostess", "hotdog", "hotel", "hound", "hour", "hourglass", "house", "houseboat", "household", "housewife", "housework", "housing", "hovel", "hovercraft", "howard", @@ -328,7 +328,7 @@ std::initializer_list nouns "laughter", "laundry", "lava", "law", "lawmaker", "lawn", "lawsuit", "lawyer", "lay", "layer", "layout", "lead", "leader", "leadership", "leading", "leaf", "league", "leaker", "leap", "learning", "leash", "leather", "leave", "leaver", "lecture", "leek", "leeway", "left", "leg", "legacy", "legal", "legend", "legging", "legislation", "legislator", "legislature", "legitimacy", "legume", "leisure", "lemon", -"lemonade", "lemur", "lender", "lending", "length", "lens", "lentil", "leopard", "leprosy", "leptocephalus", "lesbian", "lesson", "letter", +"lemonade", "lemur", "lender", "lending", "length", "lens", "lentil", "leopard", "leprosy", "leptocephalus", "lesson", "letter", "lettuce", "level", "lever", "leverage", "leveret", "liability", "liar", "liberty", "libido", "library", "licence", "license", "licensing", "licorice", "lid", "lie", "lieu", "lieutenant", "life", "lifestyle", "lifetime", "lift", "ligand", "light", "lighting", "lightning", "lightscreen", "ligula", "likelihood", "likeness", "lilac", "lily", "limb", "lime", "limestone", "limit", "limitation", "limo", "line", From f97d40584d0011644e0967a0ba8bd5c9cc595e93 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 04:10:26 +0300 Subject: [PATCH 382/390] Fix "Arcadia" --- src/Parsers/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 0a0c301b722..4ec97b8b55b 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -85,6 +85,7 @@ SRCS( MySQL/ASTDeclareReference.cpp MySQL/ASTDeclareSubPartition.cpp MySQL/ASTDeclareTableOptions.cpp + obfuscateQueries.cpp parseDatabaseAndTableName.cpp parseIdentifierOrStringLiteral.cpp parseIntervalKind.cpp From 3a2c809173c526c93ea27c7796e1d8cbbdbe8586 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 04:59:25 +0300 Subject: [PATCH 383/390] Add missing dependencies --- programs/format/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/format/CMakeLists.txt b/programs/format/CMakeLists.txt index ab06708cd3a..49f17ef163f 100644 --- a/programs/format/CMakeLists.txt +++ b/programs/format/CMakeLists.txt @@ -5,6 +5,9 @@ set (CLICKHOUSE_FORMAT_LINK boost::program_options clickhouse_common_io clickhouse_parsers + clickhouse_functions + clickhouse_aggregate_functions + clickhouse_table_functions dbms ) From c51502bce47b7d6fbcc0ab696d82d115a43b584c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 05:06:14 +0300 Subject: [PATCH 384/390] Remove words with dashes --- src/Parsers/obfuscateQueries.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 0d62fe435d8..fa7c5b4d3e4 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -103,18 +103,18 @@ std::initializer_list nouns "blowhole", "blue", "blueberry", "blush", "boar", "board", "boat", "boatload", "boatyard", "bob", "bobcat", "body", "bog", "bolero", "bolt", "bomb", "bomber", "bombing", "bond", "bonding", "bondsman", "bone", "bonfire", "bongo", "bonnet", "bonsai", "bonus", "boogeyman", "book", "bookcase", "bookend", "booking", "booklet", "bookmark", "boolean", "boom", "boon", "boost", "booster", "boot", "bootee", "bootie", -"booty", "border", "bore", "borrower", "borrowing", "bosom", "boss", "botany", "bother", "bottle", "bottling", "bottom", "bottom-line", +"booty", "border", "bore", "borrower", "borrowing", "bosom", "boss", "botany", "bother", "bottle", "bottling", "bottom", "boudoir", "bough", "boulder", "boulevard", "boundary", "bouquet", "bourgeoisie", "bout", "boutique", "bow", "bower", "bowl", "bowler", "bowling", "bowtie", "box", "boxer", "boxspring", "boy", "boycott", "boyfriend", "boyhood", "boysenberry", "bra", "brace", "bracelet", "bracket", "brain", "brake", "bran", "branch", "brand", "brandy", "brass", "brassiere", "bratwurst", "bread", "breadcrumb", "breadfruit", "break", "breakdown", "breakfast", "breakpoint", "breakthrough", "breast", "breastplate", "breath", "breeze", "brewer", "bribery", "brick", "bricklaying", "bride", "bridge", "brief", "briefing", "briefly", "briefs", "brilliant", "brink", "brisket", "broad", "broadcast", "broccoli", -"brochure", "brocolli", "broiler", "broker", "bronchitis", "bronco", "bronze", "brooch", "brood", "brook", "broom", "brother", "brother-in-law", +"brochure", "brocolli", "broiler", "broker", "bronchitis", "bronco", "bronze", "brooch", "brood", "brook", "broom", "brother", "brow", "brown", "brownie", "browser", "browsing", "brunch", "brush", "brushfire", "brushing", "bubble", "buck", "bucket", "buckle", "buckwheat", "bud", "buddy", "budget", "buffalo", "buffer", "buffet", "bug", "buggy", "bugle", "builder", "building", "bulb", "bulk", "bull", "bulldozer", "bullet", "bump", "bumper", "bun", "bunch", "bungalow", "bunghole", "bunkhouse", "burden", "bureau", "burglar", "burial", "burlesque", "burn", "burning", "burrito", "burro", "burrow", "burst", "bus", "bush", "business", "businessman", -"bust", "bustle", "butane", "butcher", "butler", "butter", "butterfly", "button", "buy", "buyer", "buying", "buzz", "buzzard", "c-clamp", +"bust", "bustle", "butane", "butcher", "butler", "butter", "butterfly", "button", "buy", "buyer", "buying", "buzz", "buzzard", "cabana", "cabbage", "cabin", "cabinet", "cable", "caboose", "cacao", "cactus", "caddy", "cadet", "cafe", "caffeine", "caftan", "cage", "cake", "calcification", "calculation", "calculator", "calculus", "calendar", "calf", "caliber", "calibre", "calico", "call", "calm", "calorie", "camel", "cameo", "camera", "camp", "campaign", "campaigning", "campanile", "camper", "campus", "can", "canal", "cancer", @@ -184,7 +184,7 @@ std::initializer_list nouns "cyst", "cytokine", "cytoplasm", "dad", "daddy", "daffodil", "dagger", "dahlia", "daikon", "daily", "dairy", "daisy", "dam", "damage", "dame", "dance", "dancer", "dancing", "dandelion", "danger", "dare", "dark", "darkness", "darn", "dart", "dash", "dashboard", "data", "database", "date", "daughter", "dawn", "day", "daybed", "daylight", "dead", "deadline", "deal", "dealer", "dealing", "dearest", -"death", "deathwatch", "debate", "debris", "debt", "debtor", "decade", "decadence", "decency", "decimal", "decision", "decision-making", +"death", "deathwatch", "debate", "debris", "debt", "debtor", "decade", "decadence", "decency", "decimal", "decision", "deck", "declaration", "declination", "decline", "decoder", "decongestant", "decoration", "decrease", "decryption", "dedication", "deduce", "deduction", "deed", "deep", "deer", "default", "defeat", "defendant", "defender", "defense", "deficit", "definition", "deformation", "degradation", "degree", "delay", "deliberation", "delight", "delivery", "demand", "democracy", "democrat", "demon", "demur", "den", @@ -210,7 +210,7 @@ std::initializer_list nouns "dress", "dresser", "dressing", "drill", "drink", "drinking", "drive", "driver", "driveway", "driving", "drizzle", "dromedary", "drop", "drudgery", "drug", "drum", "drummer", "drunk", "dryer", "duck", "duckling", "dud", "dude", "due", "duel", "dueling", "duffel", "dugout", "dulcimer", "dumbwaiter", "dump", "dump truck", "dune", "dune buggy", "dungarees", "dungeon", "duplexer", "duration", "durian", "dusk", -"dust", "dust storm", "duster", "duty", "dwarf", "dwell", "dwelling", "dynamics", "dynamite", "dynamo", "dynasty", "dysfunction", "e-book", +"dust", "dust storm", "duster", "duty", "dwarf", "dwell", "dwelling", "dynamics", "dynamite", "dynamo", "dynasty", "dysfunction", "eagle", "eaglet", "ear", "eardrum", "earmuffs", "earnings", "earplug", "earring", "earrings", "earth", "earthquake", "earthworm", "ease", "easel", "east", "eating", "eaves", "eavesdropper", "ecclesia", "echidna", "eclipse", "ecliptic", "ecology", "economics", "economy", "ecosystem", "ectoderm", "ectodermal", "ecumenist", "eddy", "edge", "edger", "edible", "editing", "edition", "editor", "editorial", @@ -273,7 +273,7 @@ std::initializer_list nouns "grill", "grin", "grip", "gripper", "grit", "grocery", "ground", "group", "grouper", "grouse", "grove", "growth", "grub", "guacamole", "guarantee", "guard", "guava", "guerrilla", "guess", "guest", "guestbook", "guidance", "guide", "guideline", "guilder", "guilt", "guilty", "guinea", "guitar", "guitarist", "gum", "gumshoe", "gun", "gunpowder", "gutter", "guy", "gym", "gymnast", "gymnastics", "gynaecology", -"gyro", "habit", "habitat", "hacienda", "hacksaw", "hackwork", "hail", "hair", "haircut", "hake", "half", "half-sister", +"gyro", "habit", "habitat", "hacienda", "hacksaw", "hackwork", "hail", "hair", "haircut", "hake", "half", "halibut", "hall", "halloween", "hallway", "halt", "ham", "hamburger", "hammer", "hammock", "hamster", "hand", "handball", "handful", "handgun", "handicap", "handle", "handlebar", "handmaiden", "handover", "handrail", "handsaw", "hanger", "happening", "happiness", "harald", "harbor", "harbour", "hardboard", "hardcover", "hardening", "hardhat", "hardship", "hardware", "hare", "harm", @@ -293,7 +293,7 @@ std::initializer_list nouns "howitzer", "hub", "hubcap", "hubris", "hug", "hugger", "hull", "human", "humanity", "humidity", "hummus", "humor", "humour", "hunchback", "hundred", "hunger", "hunt", "hunter", "hunting", "hurdle", "hurdler", "hurricane", "hurry", "hurt", "husband", "hut", "hutch", "hyacinth", "hybridisation", "hybridization", "hydrant", "hydraulics", "hydrocarb", "hydrocarbon", "hydrofoil", "hydrogen", "hydrolyse", "hydrolysis", -"hydrolyze", "hydroxyl", "hyena", "hygienic", "hype", "hyphenation", "hypochondria", "hypothermia", "hypothesis", "ice", "ice-cream", +"hydrolyze", "hydroxyl", "hyena", "hygienic", "hype", "hyphenation", "hypochondria", "hypothermia", "hypothesis", "ice", "iceberg", "icebreaker", "icecream", "icicle", "icing", "icon", "icy", "id", "idea", "ideal", "identification", "identity", "ideology", "idiom", "idiot", "igloo", "ignorance", "ignorant", "ikebana", "illegal", "illiteracy", "illness", "illusion", "illustration", "image", "imagination", "imbalance", "imitation", "immigrant", "immigration", "immortal", "impact", "impairment", "impala", "impediment", "implement", @@ -320,7 +320,7 @@ std::initializer_list nouns "kamikaze", "kangaroo", "karate", "kayak", "kazoo", "kebab", "keep", "keeper", "kendo", "kennel", "ketch", "ketchup", "kettle", "kettledrum", "key", "keyboard", "keyboarding", "keystone", "kick", "kid", "kidney", "kielbasa", "kill", "killer", "killing", "kilogram", "kilometer", "kilt", "kimono", "kinase", "kind", "kindness", "king", "kingdom", "kingfish", "kiosk", "kiss", "kit", "kitchen", "kite", -"kitsch", "kitten", "kitty", "kiwi", "knee", "kneejerk", "knickers", "knife", "knight", "knitting", "knock", "knot", "know-how", +"kitsch", "kitten", "kitty", "kiwi", "knee", "kneejerk", "knickers", "knife", "knight", "knitting", "knock", "knot", "knowledge", "knuckle", "koala", "kohlrabi", "kumquat", "lab", "label", "labor", "laboratory", "laborer", "labour", "labourer", "lace", "lack", "lacquerware", "lad", "ladder", "ladle", "lady", "ladybug", "lag", "lake", "lamb", "lambkin", "lament", "lamp", "lanai", "land", "landform", "landing", "landmine", "landscape", "lane", "language", "lantern", "lap", "laparoscope", "lapdog", "laptop", "larch", "lard", @@ -401,7 +401,7 @@ std::initializer_list nouns "paste", "pastor", "pastoralist", "pastry", "pasture", "pat", "patch", "pate", "patent", "patentee", "path", "pathogenesis", "pathology", "pathway", "patience", "patient", "patina", "patio", "patriarch", "patrimony", "patriot", "patrol", "patroller", "patrolling", "patron", "pattern", "patty", "pattypan", "pause", "pavement", "pavilion", "paw", "pawnshop", "pay", "payee", "payment", "payoff", "pea", "peace", -"peach", "peacoat", "peacock", "peak", "peanut", "pear", "pearl", "peasant", "pecan", "pecker", "pedal", "peek", "peen", "peer", "peer-to-peer", +"peach", "peacoat", "peacock", "peak", "peanut", "pear", "pearl", "peasant", "pecan", "pecker", "pedal", "peek", "peen", "peer", "pegboard", "pelican", "pelt", "pen", "penalty", "pence", "pencil", "pendant", "pendulum", "penguin", "penicillin", "peninsula", "penis", "pennant", "penny", "pension", "pentagon", "peony", "people", "pepper", "pepperoni", "percent", "percentage", "perception", "perch", "perennial", "perfection", "performance", "perfume", "period", "periodical", "peripheral", "permafrost", "permission", "permit", "perp", @@ -485,7 +485,7 @@ std::initializer_list nouns "setback", "setting", "settlement", "settler", "severity", "sewer", "sex", "sexuality", "shack", "shackle", "shade", "shadow", "shadowbox", "shakedown", "shaker", "shallot", "shallows", "shame", "shampoo", "shanty", "shape", "share", "shareholder", "shark", "shaw", "shawl", "shear", "shearling", "sheath", "shed", "sheep", "sheet", "shelf", "shell", "shelter", "sherbet", "sherry", "shield", "shift", "shin", -"shine", "shingle", "ship", "shipper", "shipping", "shipyard", "shirt", "shirtdress", "shoat", "shock", "shoe", "shoe-horn", +"shine", "shingle", "ship", "shipper", "shipping", "shipyard", "shirt", "shirtdress", "shoat", "shock", "shoe", "shoehorn", "shoelace", "shoemaker", "shoes", "shoestring", "shofar", "shoot", "shootdown", "shop", "shopper", "shopping", "shore", "shoreline", "short", "shortage", "shorts", "shortwave", "shot", "shoulder", "shout", "shovel", "show", "shower", "shred", "shrimp", "shrine", "shutdown", "sibling", "sick", "sickness", "side", "sideboard", "sideburns", "sidecar", "sidestream", "sidewalk", "siding", @@ -513,7 +513,7 @@ std::initializer_list nouns "staff", "stag", "stage", "stain", "stair", "staircase", "stake", "stalk", "stall", "stallion", "stamen", "stamina", "stamp", "stance", "stand", "standard", "standardisation", "standardization", "standing", "standoff", "standpoint", "star", "starboard", "start", "starter", "state", "statement", "statin", "station", "statistic", "statistics", "statue", "status", "statute", "stay", "steak", -"stealth", "steam", "steamroller", "steel", "steeple", "stem", "stench", "stencil", "step", "step-daughter", +"stealth", "steam", "steamroller", "steel", "steeple", "stem", "stench", "stencil", "step", "stepdaughter", "stepmother", "stepson", "stereo", "stew", "steward", "stick", "sticker", "stiletto", "still", "stimulation", "stimulus", "sting", "stinger", "stitch", "stitcher", "stock", "stockings", "stole", "stomach", "stone", "stonework", "stool", @@ -533,7 +533,7 @@ std::initializer_list nouns "swivel", "sword", "swordfight", "swordfish", "sycamore", "symbol", "symmetry", "sympathy", "symptom", "syndicate", "syndrome", "synergy", "synod", "synonym", "synthesis", "syrup", "system", "tab", "tabby", "tabernacle", "table", "tablecloth", "tablet", "tabletop", "tachometer", "tackle", "taco", "tactics", "tactile", "tadpole", "tag", "tail", "tailbud", "tailor", "tailspin", "takeover", -"tale", "talent", "talk", "talking", "tamale", "tambour", "tambourine", "tan", "tandem", "tangerine", "tank", "tank-top", +"tale", "talent", "talk", "talking", "tamale", "tambour", "tambourine", "tan", "tandem", "tangerine", "tank", "tanker", "tankful", "tap", "tape", "tapioca", "target", "taro", "tarragon", "tart", "task", "tassel", "taste", "tatami", "tattler", "tattoo", "tavern", "tax", "taxi", "taxicab", "taxpayer", "tea", "teacher", "teaching", "team", "teammate", "teapot", "tear", "tech", "technician", "technique", "technologist", "technology", "tectonics", "teen", "teenager", "teepee", "telephone", "telescreen", "teletype", From f3349c8d138e882793e01f8708b7e0fadb7ad937 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 05:13:20 +0300 Subject: [PATCH 385/390] Keywords are case-insensitive --- src/Parsers/obfuscateQueries.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index fa7c5b4d3e4..3594bdef3eb 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -884,7 +885,10 @@ void obfuscateQueries( if (token.type == TokenType::BareWord) { - if (keywords.count(whole_token) + std::string whole_token_uppercase(whole_token); + Poco::toUpperInPlace(whole_token_uppercase); + + if (keywords.count(whole_token_uppercase) || known_identifier_func(whole_token)) { /// Keep keywords as is. From 11ab250a712c686b6fee7ea14df60d2c57634bf3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 05:18:18 +0300 Subject: [PATCH 386/390] More test cases --- tests/queries/0_stateless/01508_query_obfuscator.reference | 3 ++- tests/queries/0_stateless/01508_query_obfuscator.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01508_query_obfuscator.reference b/tests/queries/0_stateless/01508_query_obfuscator.reference index e2f61cf345d..7066528c870 100644 --- a/tests/queries/0_stateless/01508_query_obfuscator.reference +++ b/tests/queries/0_stateless/01508_query_obfuscator.reference @@ -1 +1,2 @@ -SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Meantime_id_brisketHeavy, exclamation(Grit), avgIf(remote('128.0.0.1')) +SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Residue_id_breakfastDevice, park(Innervation), avgIf(remote('128.0.0.1')) +SELECT shell_dust_tintype between crumb and shoat, case when peach >= 116 then bombing else null end diff --git a/tests/queries/0_stateless/01508_query_obfuscator.sh b/tests/queries/0_stateless/01508_query_obfuscator.sh index 797271edb30..a5dd30b67ba 100755 --- a/tests/queries/0_stateless/01508_query_obfuscator.sh +++ b/tests/queries/0_stateless/01508_query_obfuscator.sh @@ -4,3 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT 123, 'Test://2020-01-01hello1234 at 2000-01-01T01:02:03', 12e100, Gibberish_id_testCool, hello(World), avgIf(remote('127.0.0.1'))" +$CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT cost_first_screen between a and b, case when x >= 123 then y else null end" + From 5e146eda0403a781f78191f11b2a4f835d0b1307 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 06:02:05 +0300 Subject: [PATCH 387/390] Fix error --- src/Common/BitHelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/BitHelpers.h b/src/Common/BitHelpers.h index eac5fdac80e..e79daeba14e 100644 --- a/src/Common/BitHelpers.h +++ b/src/Common/BitHelpers.h @@ -66,7 +66,7 @@ inline size_t getLeadingZeroBits(T x) template inline uint32_t bitScanReverse(T x) { - return sizeof(T) * 8 - 1 - getLeadingZeroBitsUnsafe(x); + return (std::max(sizeof(T), sizeof(unsigned int))) * 8 - 1 - getLeadingZeroBitsUnsafe(x); } // Unsafe since __builtin_ctz()-family explicitly state that result is undefined on x == 0 From 57b480e65d5713198e0bd81e8cf84564d4e7d1b0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 07:03:55 +0300 Subject: [PATCH 388/390] Fix PVS warning --- programs/client/Suggest.cpp | 2 +- src/Parsers/obfuscateQueries.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp index ac18a131c3a..e85e7a21261 100644 --- a/programs/client/Suggest.cpp +++ b/programs/client/Suggest.cpp @@ -80,7 +80,7 @@ Suggest::Suggest() "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", - "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "FOR", "RANDOMIZED", + "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE"}; } diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 3594bdef3eb..32382b70bd7 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -37,7 +37,7 @@ const std::unordered_set keywords "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", - "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "FOR", "RANDOMIZED", + "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY" }; From da8a93806921085fdd6fda50fdf2db84f53e74e4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 26 Sep 2020 09:50:34 +0300 Subject: [PATCH 389/390] Replace unit test to functional test --- src/CMakeLists.txt | 2 +- src/Parsers/tests/gtest_obfuscate_queries.cpp | 92 ------------------- .../01508_query_obfuscator.reference | 14 +++ .../0_stateless/01508_query_obfuscator.sh | 14 +++ 4 files changed, 29 insertions(+), 93 deletions(-) delete mode 100644 src/Parsers/tests/gtest_obfuscate_queries.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 40815228eac..0016c51b7f8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -415,6 +415,6 @@ if (ENABLE_TESTS AND USE_GTEST) -Wno-gnu-zero-variadic-macro-arguments ) - target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) + target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) add_check(unit_tests_dbms) endif () diff --git a/src/Parsers/tests/gtest_obfuscate_queries.cpp b/src/Parsers/tests/gtest_obfuscate_queries.cpp deleted file mode 100644 index ada15109d17..00000000000 --- a/src/Parsers/tests/gtest_obfuscate_queries.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -using namespace DB; - - -TEST(ObfuscateQueries, Test1) -{ - WordMap obfuscated_words_map; - WordSet used_nouns; - SipHash hash_func; - - std::string salt = "Hello, world"; - hash_func.update(salt); - - SharedContextHolder shared_context; - const ContextHolder & context_holder = getContext(); - - registerFunctions(); - registerAggregateFunctions(); - registerTableFunctions(); - registerStorages(); - - std::unordered_set additional_names; - - auto all_known_storage_names = StorageFactory::instance().getAllRegisteredNames(); - auto all_known_data_type_names = DataTypeFactory::instance().getAllRegisteredNames(); - - additional_names.insert(all_known_storage_names.begin(), all_known_storage_names.end()); - additional_names.insert(all_known_data_type_names.begin(), all_known_data_type_names.end()); - - KnownIdentifierFunc is_known_identifier = [&](std::string_view name) - { - std::string what(name); - - return FunctionFactory::instance().tryGet(what, context_holder.context) != nullptr - || AggregateFunctionFactory::instance().isAggregateFunctionName(what) - || TableFunctionFactory::instance().isTableFunctionName(what) - || additional_names.count(what); - }; - - WriteBufferFromOwnString out; - - obfuscateQueries( - R"( -SELECT - VisitID, - Goals.ID, Goals.EventTime, - WatchIDs, - EAction.ProductName, EAction.ProductPrice, EAction.ProductCurrency, EAction.ProductQuantity, EAction.EventTime, EAction.Type -FROM merge.visits_v2 -WHERE - StartDate >= '2020-09-17' AND StartDate <= '2020-09-25' - AND CounterID = 24226447 - AND intHash32(UserID) = 416638616 AND intHash64(UserID) = 13269091395366875299 - AND VisitID IN (5653048135597886819, 5556254872710352304, 5516214175671455313, 5476714937521999313, 5464051549483503043) - AND Sign = 1 -)", - out, obfuscated_words_map, used_nouns, hash_func, is_known_identifier); - - EXPECT_EQ(out.str(), R"( -SELECT - CorduroyID, - Steel.ID, Steel.AcornSidestream, - WealthBRANCH, - GOVERNMENT.SedimentName, GOVERNMENT.SedimentExhaustion, GOVERNMENT.SedimentFencing, GOVERNMENT.SedimentOpossum, GOVERNMENT.AcornSidestream, GOVERNMENT.Lute -FROM merge.luncheonette_pants -WHERE - GovernanceCreche >= '2021-04-16' AND GovernanceCreche <= '2021-04-24' - AND StarboardID = 26446940 - AND intHash32(MessyID) = 474525514 AND intHash64(MessyID) = 13916317227779800149 - AND CorduroyID IN (5223158832904664474, 5605365157729463108, 7543250143731591192, 8715842063486405567, 7837015536326316923) - AND Tea = 1 -)"); -} - diff --git a/tests/queries/0_stateless/01508_query_obfuscator.reference b/tests/queries/0_stateless/01508_query_obfuscator.reference index 7066528c870..0064ac73a09 100644 --- a/tests/queries/0_stateless/01508_query_obfuscator.reference +++ b/tests/queries/0_stateless/01508_query_obfuscator.reference @@ -1,2 +1,16 @@ SELECT 116, 'Qqfu://2020-02-10isqkc1203 sp 2000-05-27T18:38:01', 13e100, Residue_id_breakfastDevice, park(Innervation), avgIf(remote('128.0.0.1')) SELECT shell_dust_tintype between crumb and shoat, case when peach >= 116 then bombing else null end + +SELECT + ChimeID, + Testimonial.ID, Testimonial.SipCauseway, + TankfulTRUMPET, + HUMIDITY.TermiteName, HUMIDITY.TermiteSculptural, HUMIDITY.TermiteGuilt, HUMIDITY.TermiteIntensity, HUMIDITY.SipCauseway, HUMIDITY.Coat +FROM merge.tinkle_efficiency +WHERE + FaithSeller >= '2020-10-13' AND FaithSeller <= '2020-10-21' + AND MandolinID = 30750384 + AND intHash32(GafferID) = 448362928 AND intHash64(GafferID) = 12572659331310383983 + AND ChimeID IN (8195672321757027078, 7079643623150622129, 5057006826979676478, 7886875230160484653, 7494974311229040743) + AND Stot = 1 + diff --git a/tests/queries/0_stateless/01508_query_obfuscator.sh b/tests/queries/0_stateless/01508_query_obfuscator.sh index a5dd30b67ba..d60e42489fa 100755 --- a/tests/queries/0_stateless/01508_query_obfuscator.sh +++ b/tests/queries/0_stateless/01508_query_obfuscator.sh @@ -6,3 +6,17 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT 123, 'Test://2020-01-01hello1234 at 2000-01-01T01:02:03', 12e100, Gibberish_id_testCool, hello(World), avgIf(remote('127.0.0.1'))" $CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< "SELECT cost_first_screen between a and b, case when x >= 123 then y else null end" +$CLICKHOUSE_FORMAT --seed Hello --obfuscate <<< " +SELECT + VisitID, + Goals.ID, Goals.EventTime, + WatchIDs, + EAction.ProductName, EAction.ProductPrice, EAction.ProductCurrency, EAction.ProductQuantity, EAction.EventTime, EAction.Type +FROM merge.visits_v2 +WHERE + StartDate >= '2020-09-17' AND StartDate <= '2020-09-25' + AND CounterID = 24226447 + AND intHash32(UserID) = 416638616 AND intHash64(UserID) = 13269091395366875299 + AND VisitID IN (5653048135597886819, 5556254872710352304, 5516214175671455313, 5476714937521999313, 5464051549483503043) + AND Sign = 1 +" From d1bb6b655553d429dd47dbc2079f8de26a4a238c Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 26 Sep 2020 11:47:26 +0300 Subject: [PATCH 390/390] Update test.py --- tests/integration/test_s3_with_proxy/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 3d118266455..70a50ae0e15 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -40,7 +40,7 @@ def cluster(): def check_proxy_logs(cluster, proxy_instance, http_methods={"POST", "PUT", "GET", "DELETE"}): for i in range(10): logs = cluster.get_container_logs(proxy_instance) - # Check that all possible interactions with Minio are present + # Check with retry that all possible interactions with Minio are present for http_method in http_methods: if logs.find(http_method + " http://minio1") >= 0: return