From aa5e60a23bfbb40a80d2ffc0fa0bee00c52b7cca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Feb 2019 17:05:48 +0300 Subject: [PATCH 01/13] Fixed bad configuration of base64 library #3350 --- contrib/base64-cmake/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/base64-cmake/CMakeLists.txt b/contrib/base64-cmake/CMakeLists.txt index 9357423c992..8ec83201109 100644 --- a/contrib/base64-cmake/CMakeLists.txt +++ b/contrib/base64-cmake/CMakeLists.txt @@ -39,5 +39,10 @@ add_library(base64 ${LINK_MODE} ${LIBRARY_DIR}/lib/codecs.h ${CMAKE_CURRENT_BINARY_DIR}/config.h) -target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt}) +set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx/codec.c PROPERTIES COMPILE_FLAGS -mavx) +set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx2/codec.c PROPERTIES COMPILE_FLAGS -mavx2) +set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse41/codec.c PROPERTIES COMPILE_FLAGS -msse4.1) +set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse42/codec.c PROPERTIES COMPILE_FLAGS -msse4.2) +set_source_files_properties(${LIBRARY_DIR}/lib/arch/ssse3/codec.c PROPERTIES COMPILE_FLAGS -mssse3) + target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}) From 9dd2e75b71461dba937f3cea44baad495db6514f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Feb 2019 19:43:52 +0300 Subject: [PATCH 02/13] Fixed warnings in clang 8 #4143 --- .../Storages/MergeTree/MergeTreeMinMaxIndex.cpp | 6 +++--- .../Storages/MergeTree/MergeTreeMinMaxIndex.h | 14 +++++++------- .../Storages/MergeTree/MergeTreeUniqueIndex.cpp | 2 +- .../Storages/MergeTree/MergeTreeUniqueIndex.h | 16 ++++++++-------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp index d15e86a6e22..3287632d39a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.cpp @@ -140,11 +140,11 @@ std::unique_ptr MergeTreeMinMaxIndexCreator( ASTPtr expr_list = MergeTreeData::extractKeyExpressionList(node->expr->clone()); auto syntax = SyntaxAnalyzer(context, {}).analyze( - expr_list, new_columns); + expr_list, new_columns); auto minmax_expr = ExpressionAnalyzer(expr_list, syntax, context).getActions(false); auto sample = ExpressionAnalyzer(expr_list, syntax, context) - .getActions(true)->getSampleBlock(); + .getActions(true)->getSampleBlock(); Names columns; DataTypes data_types; @@ -158,7 +158,7 @@ std::unique_ptr MergeTreeMinMaxIndexCreator( } return std::make_unique( - node->name, std::move(minmax_expr), columns, data_types, sample, node->granularity.get());; + node->name, std::move(minmax_expr), columns, data_types, sample, node->granularity.get()); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h index ac1ca1787e1..2e41a0cf966 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMinMaxIndex.h @@ -55,13 +55,13 @@ class MergeTreeMinMaxIndex : public MergeTreeIndex { public: MergeTreeMinMaxIndex( - String name, - ExpressionActionsPtr expr, - const Names & columns, - const DataTypes & data_types, - const Block & header, - size_t granularity) - : MergeTreeIndex(name, expr, columns, data_types, header, granularity) {} + String name_, + ExpressionActionsPtr expr_, + const Names & columns_, + const DataTypes & data_types_, + const Block & header_, + size_t granularity_) + : MergeTreeIndex(name_, expr_, columns_, data_types_, header_, granularity_) {} ~MergeTreeMinMaxIndex() override = default; diff --git a/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.cpp b/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.cpp index 0a29a7fbd07..5db8c5aa89d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.cpp @@ -387,7 +387,7 @@ std::unique_ptr MergeTreeUniqueIndexCreator( } return std::make_unique( - node->name, std::move(unique_expr), columns, data_types, header, node->granularity.get(), max_rows);; + node->name, std::move(unique_expr), columns, data_types, header, node->granularity.get(), max_rows); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.h b/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.h index 6b1107dffd6..8f371a64a71 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.h +++ b/dbms/src/Storages/MergeTree/MergeTreeUniqueIndex.h @@ -68,14 +68,14 @@ class MergeTreeUniqueIndex : public MergeTreeIndex { public: MergeTreeUniqueIndex( - String name, - ExpressionActionsPtr expr, - const Names & columns, - const DataTypes & data_types, - const Block & header, - size_t granularity, - size_t _max_rows) - : MergeTreeIndex(std::move(name), std::move(expr), columns, data_types, header, granularity), max_rows(_max_rows) {} + String name_, + ExpressionActionsPtr expr_, + const Names & columns_, + const DataTypes & data_types_, + const Block & header_, + size_t granularity_, + size_t max_rows_) + : MergeTreeIndex(std::move(name_), std::move(expr_), columns_, data_types_, header_, granularity_), max_rows(max_rows_) {} ~MergeTreeUniqueIndex() override = default; From 693782ee5b956582174b7e7e0e43553916373b23 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 5 Feb 2019 19:46:30 +0300 Subject: [PATCH 03/13] WIP on docs/website (#4278) --- docs/en/interfaces/third-party/integrations.md | 2 ++ docs/en/operations/index.md | 14 ++++++++++++++ docs/fa/interfaces/third-party/integrations.md | 2 ++ docs/ru/interfaces/third-party/integrations.md | 2 ++ docs/ru/operations/index.md | 15 +++++++++++++++ docs/toc_en.yml | 2 +- docs/toc_fa.yml | 2 +- docs/toc_ru.yml | 2 +- docs/toc_zh.yml | 2 +- docs/zh/interfaces/third-party/integrations.md | 2 ++ website/index.html | 2 +- 11 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 02ab7ba92a0..76833a869f6 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -36,6 +36,8 @@ - [Nagios](https://www.nagios.org/) - [check_clickhouse](https://github.com/exogroup/check_clickhouse/) - Logging + - [rsyslog](https://www.rsyslog.com/) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - [loghouse](https://github.com/flant/loghouse) (for [Kubernetes](https://kubernetes.io)) diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md index 63cb19bb639..399a180ec46 100644 --- a/docs/en/operations/index.md +++ b/docs/en/operations/index.md @@ -1,4 +1,18 @@ # Operations +ClickHouse operations manual consists of the following major sections: + + - [Requirements](requirements.md) + - [Monitoring](monitoring.md) + - [Troubleshooting](troubleshooting.md) + - [Usage Recommendations](tips.md) + - [Access Rights](access_rights.md) + - [Data Backup](backup.md) + - [Configuration Files](configuration_files.md) + - [Quotas](quotas.md) + - [System Tables](system_tables.md) + - [Server Configuration Parameters](server_settings/index.md) + - [Settings](settings/index.md) + - [Utilities](utils/index.md) [Original article](https://clickhouse.yandex/docs/en/operations/) diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md index 5a648df8f1a..d0b2e041799 100644 --- a/docs/fa/interfaces/third-party/integrations.md +++ b/docs/fa/interfaces/third-party/integrations.md @@ -37,6 +37,8 @@ - [Nagios](https://www.nagios.org/) - [check_clickhouse](https://github.com/exogroup/check_clickhouse/) - ثبت نام + - [rsyslog](https://www.rsyslog.com/) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - [loghouse](https://github.com/flant/loghouse) (برای [Kubernetes](https://kubernetes.io)) diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 4235add28df..ee9864a16b7 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -35,6 +35,8 @@ - [Nagios](https://www.nagios.org/) - [check_clickhouse](https://github.com/exogroup/check_clickhouse/) - Логирование + - [rsyslog](https://www.rsyslog.com/) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - [loghouse](https://github.com/flant/loghouse) (для [Kubernetes](https://kubernetes.io)) diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index f16d6b3f8d7..5e691e81047 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,3 +1,18 @@ # Эксплуатация +Руководство по эксплуатации ClickHouse состоит из следующих основных разделов: + + - [Требования](requirements.md) + - [Мониторинг](monitoring.md) + - [Решение проблем](troubleshooting.md) + - [Советы по эксплуатации](tips.md) + - [Права доступа](access_rights.md) + - [Резервное копирование](backup.md) + - [Конфигурационные файлы](configuration_files.md) + - [Квоты](quotas.md) + - [Системные таблицы](system_tables.md) + - [Конфигурационные параметры сервера](server_settings/index.md) + - [Настройки](settings/index.md) + - [Утилиты](utils/index.md) + [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 598a0af2d74..f41e94b1a56 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -157,7 +157,7 @@ nav: - 'General Syntax': 'query_language/syntax.md' - 'Operations': - - 'hidden': 'operations/index.md' + - 'Introduction': 'operations/index.md' - 'Requirements': 'operations/requirements.md' - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index f911c9850ec..16a4b2f729f 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -157,7 +157,7 @@ nav: - 'General Syntax': 'query_language/syntax.md' - 'Operations': - - 'hidden': 'operations/index.md' + - 'Introduction': 'operations/index.md' - 'Requirements': 'operations/requirements.md' - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 6f3ab12be4b..3b588f37896 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -155,7 +155,7 @@ nav: - 'Общий синтаксис': 'query_language/syntax.md' - 'Эксплуатация': - - 'hidden': 'operations/index.md' + - 'Введение': 'operations/index.md' - 'Требования': 'operations/requirements.md' - 'Мониторинг': 'operations/monitoring.md' - 'Решение проблем': 'operations/troubleshooting.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index d9cc2e77054..ed4f3da3fe2 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -156,7 +156,7 @@ nav: - 'General syntax': 'query_language/syntax.md' - '运维': - - 'hidden': 'operations/index.md' + - 'Introduction': 'operations/index.md' - 'Requirements': 'operations/requirements.md' - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 1a42b45b901..6c77f6bb1e7 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -34,6 +34,8 @@ - [Nagios](https://www.nagios.org/) - [check_clickhouse](https://github.com/exogroup/check_clickhouse/) - 记录 + - [rsyslog](https://www.rsyslog.com/) + - [omclickhouse](https://www.rsyslog.com/doc/master/configuration/modules/omclickhouse.html) - [fluentd](https://www.fluentd.org) - [loghouse](https://github.com/flant/loghouse) (对于 [Kubernetes](https://kubernetes.io)) diff --git a/website/index.html b/website/index.html index 0fd3cc40d33..aa809a67210 100644 --- a/website/index.html +++ b/website/index.html @@ -94,7 +94,7 @@
- C++ ClickHouse and CatBoost Sprints in Moscow on February 2 + Upcoming ClickHouse Community Meetups: San Francisco on February 19 and Madrid on April 2
From 396d4fb921bd9061b5d1dfd13da227714870dda7 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 5 Feb 2019 19:58:57 +0300 Subject: [PATCH 04/13] hotfix for duplicates in JOIN ON #4271 --- dbms/src/Interpreters/Join.cpp | 8 +++++++- .../0_stateless/00818_join_bug_4271.reference | 9 +++++++++ .../0_stateless/00818_join_bug_4271.sql | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00818_join_bug_4271.reference create mode 100644 dbms/tests/queries/0_stateless/00818_join_bug_4271.sql diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 4b7731b2e42..8858dd0f2b6 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -469,9 +469,15 @@ bool Join::insertFromBlock(const Block & block) } else { + NameSet erased; /// HOTFIX: there could be duplicates in JOIN ON section + /// Remove the key columns from stored_block, as they are not needed. for (const auto & name : key_names_right) - stored_block->erase(stored_block->getPositionByName(name)); + { + if (!erased.count(name)) + stored_block->erase(stored_block->getPositionByName(name)); + erased.insert(name); + } } size_t size = stored_block->columns(); diff --git a/dbms/tests/queries/0_stateless/00818_join_bug_4271.reference b/dbms/tests/queries/0_stateless/00818_join_bug_4271.reference new file mode 100644 index 00000000000..4d591381421 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00818_join_bug_4271.reference @@ -0,0 +1,9 @@ +1 1 a 1 1 a +2 2 b \N \N \N +1 1 a 1 1 a +2 2 b \N \N \N +1 1 a 1 1 a +1 1 a 1 1 a +2 2 b \N \N \N +1 1 a 1 1 a +2 2 b \N \N \N diff --git a/dbms/tests/queries/0_stateless/00818_join_bug_4271.sql b/dbms/tests/queries/0_stateless/00818_join_bug_4271.sql new file mode 100644 index 00000000000..8f7fc7f8ef4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00818_join_bug_4271.sql @@ -0,0 +1,19 @@ +use test; + +drop table if exists t; +drop table if exists s; + +create table t(a Nullable(Int64), b Nullable(Int64), c Nullable(String)) engine = Memory; +create table s(a Nullable(Int64), b Nullable(Int64), c Nullable(String)) engine = Memory; + +insert into t values(1,1,'a'), (2,2,'b'); +insert into s values(1,1,'a'); + +select * from t left join s on t.a = s.a; +select * from t left join s on t.a = s.a and t.a = s.b; +select * from t left join s on t.a = s.a where s.a = 1; +select * from t left join s on t.a = s.a and t.a = s.a; +select * from t left join s on t.a = s.a and t.b = s.a; + +drop table t; +drop table s; From 238cfb4da342914a3680602a62d27ff9788a7c39 Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 5 Feb 2019 20:05:23 +0300 Subject: [PATCH 05/13] CLICKHOUSE-4284 debian init: wait for server startup on 'clickhouse-server start' --- debian/clickhouse-server.init | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 2c72d7322d4..28f8481aff7 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -174,7 +174,16 @@ start() fi if [ $EXIT_STATUS -eq 0 ]; then - echo "DONE" + attempts=0 + while ! is_running && [ $attempts -le 10 ]; do + attempts=$(($attempts + 1)) + sleep 1 + done + if is_running; then + echo "DONE" + else + echo "UNKNOWN" + fi else echo "FAILED" fi From 607951895a7829e594cf2643dc6582dea2598a6c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Feb 2019 20:05:33 +0300 Subject: [PATCH 06/13] Fixed table function numbers with zero argument #4276 --- dbms/src/Storages/System/StorageSystemNumbers.cpp | 8 ++++---- dbms/src/Storages/System/StorageSystemNumbers.h | 6 +++--- .../00836_numbers_table_function_zero.reference | 15 +++++++++++++++ .../00836_numbers_table_function_zero.sql | 8 ++++++++ 4 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.reference create mode 100644 dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.sql diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index dbf5bb54783..f0bd4cfb0ff 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -43,7 +43,7 @@ private: }; -StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_, size_t offset_) +StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, std::optional limit_, size_t offset_) : name(name_), multithreaded(multithreaded_), limit(limit_), offset(offset_) { setColumns(ColumnsDescription({{"number", std::make_shared()}})); @@ -60,9 +60,9 @@ BlockInputStreams StorageSystemNumbers::read( { check(column_names); - if (limit && limit < max_block_size) + if (limit && *limit < max_block_size) { - max_block_size = std::min(max_block_size, limit); + max_block_size = std::min(max_block_size, *limit); multithreaded = false; } @@ -75,7 +75,7 @@ BlockInputStreams StorageSystemNumbers::read( res[i] = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); if (limit) /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. - res[i] = std::make_shared(res[i], limit * (i + 1) / num_streams - limit * i / num_streams, 0); + res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0); } return res; diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 926e336e162..d8465343be6 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -36,12 +37,11 @@ public: private: const std::string name; bool multithreaded; - size_t limit; + std::optional limit; size_t offset; protected: - /// limit: 0 means unlimited. - StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_ = 0, size_t offset_ = 0); + StorageSystemNumbers(const std::string & name_, bool multithreaded_, std::optional limit_ = std::nullopt, size_t offset_ = 0); }; } diff --git a/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.reference b/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.reference new file mode 100644 index 00000000000..f86671f9eb9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.reference @@ -0,0 +1,15 @@ +0 +1 +10 +100000 +0 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.sql b/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.sql new file mode 100644 index 00000000000..e70086e64de --- /dev/null +++ b/dbms/tests/queries/0_stateless/00836_numbers_table_function_zero.sql @@ -0,0 +1,8 @@ +SELECT count() FROM numbers(0); +SELECT count() FROM numbers(1); +SELECT count() FROM numbers(10); +SELECT count() FROM numbers(100000); + +SELECT * FROM numbers(0); +SELECT * FROM numbers(1); +SELECT * FROM numbers(10); From 857b5abe874dcc2cbdcb0d98fdcbde046e5d793d Mon Sep 17 00:00:00 2001 From: KochetovNicolai Date: Tue, 5 Feb 2019 21:33:52 +0300 Subject: [PATCH 07/13] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72071111672..97519601afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * Added functions `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` for SQL standard compatibility. [#3826](https://github.com/yandex/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) * Support for write in `HDFS` tables and `hdfs` table function. [#4084](https://github.com/yandex/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin)) * Added functions to search for multiple constant strings from big haystack: `multiPosition`, `multiSearch` ,`firstMatch` also with `-UTF8`, `-CaseInsensitive`, and `-CaseInsensitiveUTF8` variants. [#4053](https://github.com/yandex/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) -* Pruning of unused shards if `SELECT` query filters by sharding key (setting `distributed_optimize_skip_select_on_unused_shards`). [#3851](https://github.com/yandex/ClickHouse/pull/3851) ([Ivan](https://github.com/abyss7)) +* Pruning of unused shards if `SELECT` query filters by sharding key (setting `distributed_optimize_skip_select_on_unused_shards`). [#3851](https://github.com/yandex/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [Ivan](https://github.com/abyss7)) * Allow `Kafka` engine to ignore some number of parsing errors per block. [#4094](https://github.com/yandex/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7)) * Added support for `CatBoost` multiclass models evaluation. Function `modelEvaluate` returns tuple with per-class raw predictions for multiclass models. `libcatboostmodel.so` should be built with [#607](https://github.com/catboost/catboost/pull/607). [#3959](https://github.com/yandex/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) * Added functions `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [#4097](https://github.com/yandex/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) From 21eaf021f9803c6af2fb8bbf58208568fae60990 Mon Sep 17 00:00:00 2001 From: KochetovNicolai Date: Tue, 5 Feb 2019 22:36:40 +0300 Subject: [PATCH 08/13] Update CHANGELOG_RU.md --- CHANGELOG_RU.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index fa438686132..ac86e1a5cd1 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,3 +1,112 @@ +## ClickHouse release 19.1.6, 2019-01-24 + +### Новые возможности: + +* Задание формата сжатия для отдельных столбцов. [#3899](https://github.com/yandex/ClickHouse/pull/3899) [#4111](https://github.com/yandex/ClickHouse/pull/4111) ([alesapin](https://github.com/alesapin), [Winter Zhang](https://github.com/zhang2014), [Anatoly](https://github.com/Sindbag)) +* Формат сжатия `Delta`. [#4052](https://github.com/yandex/ClickHouse/pull/4052) ([alesapin](https://github.com/alesapin)) +* Изменение формата сжатия запросом `ALTER`. [#4054](https://github.com/yandex/ClickHouse/pull/4054) ([alesapin](https://github.com/alesapin)) +* Добавлены функции `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` для совместимости со стандартом SQL. [#3826](https://github.com/yandex/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov)) +* Поддержка записи в движок `HDFS` и табличную функцию `hdfs`. [#4084](https://github.com/yandex/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin)) +* Добавлены функции поиска набора константных строк в тексте: `multiPosition`, `multiSearch` ,`firstMatch` также с суффиксами `-UTF8`, `-CaseInsensitive`, и `-CaseInsensitiveUTF8`. [#4053](https://github.com/yandex/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1)) +* Пропуск неиспользуемых шардов в случае, если запрос `SELECT` содержит фильтрацию по ключу шардирования (настройка `distributed_optimize_skip_select_on_unused_shards`). [#3851](https://github.com/yandex/ClickHouse/pull/3851) ([Gleb Kanterov](https://github.com/kanterov), [Ivan](https://github.com/abyss7)) +* Пропуск строк в случае ошибки парсинга для движка `Kafka` (настройка `kafka_skip_broken_messages`). [#4094](https://github.com/yandex/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7)) +* Поддержка применения мультиклассовых моделей `CatBoost`. Функция `modelEvaluate` возвращает кортеж в случае использования мультиклассовой модели. `libcatboostmodel.so` should be built with [#607](https://github.com/catboost/catboost/pull/607). [#3959](https://github.com/yandex/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Добавлены функции `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [#4097](https://github.com/yandex/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea)) +* Добавлены функции хеширования `xxHash64` и `xxHash32`. [#3905](https://github.com/yandex/ClickHouse/pull/3905) ([filimonov](https://github.com/filimonov)) +* Добавлена функция хеширования `gccMurmurHash` (GCC flavoured Murmur hash), использующая те же hash seed, что и [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [#4000](https://github.com/yandex/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li)) +* Добавлены функции хеширования `javaHash`, `hiveHash`. [#3811](https://github.com/yandex/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365)) +* Добавлена функция `remoteSecure`. Функция работает аналогично `remote`, но использует безопасное соединение. [#4088](https://github.com/yandex/ClickHouse/pull/4088) ([proller](https://github.com/proller)) + + +### Экспериментальные возможности: + +* Эмуляция запросов с несколькими секциями `JOIN` (настройка `allow_experimental_multiple_joins_emulation`). [#3946](https://github.com/yandex/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2)) + +### Исправления ошибок: + +* Ограничен размер кеша скомпилированных выражений в случае, если не указана настройка `compiled_expression_cache_size` для экономии потребляемой памяти. [#4041](https://github.com/yandex/ClickHouse/pull/4041) ([alesapin](https://github.com/alesapin)) +* Исправлена проблема зависания потоков, выполняющих запрос `ALTER` для таблиц семейства `Replicated`, а также потоков, обновляющих конфигурацию из ZooKeeper. [#2947](https://github.com/yandex/ClickHouse/issues/2947) [#3891](https://github.com/yandex/ClickHouse/issues/3891) [#3934](https://github.com/yandex/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn)) +* Исправлен race condition в случае выполнения распределенной задачи запроса `ALTER`. Race condition приводил к состоянию, когда более чем одна реплика пыталась выполнить задачу, в результате чего все такие реплики, кроме одной, падали с ошибкой обращения к ZooKeeper. [#3904](https://github.com/yandex/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn)) +* Исправлена проблема обновления настройки `from_zk`. Настройка, указанная в файле конфигурации, не обновлялась в случае, если запрос к ZooKeeper падал по timeout. [#2947](https://github.com/yandex/ClickHouse/issues/2947) [#3947](https://github.com/yandex/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn)) +* Исправлена ошибка в вычислении сетевого префикса при указании IPv4 маски подсети. [#3945](https://github.com/yandex/ClickHouse/pull/3945) ([alesapin](https://github.com/alesapin)) +* Исправлено падение (`std::terminate`) в редком сценарии, когда новый поток не мог быть создан из-за нехватки ресурсов. [#3956](https://github.com/yandex/ClickHouse/pull/3956) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлено падение табличной функции `remote` в случае, когда не удавалось получить структуру таблицы из-за ограничений пользователя. [#4009](https://github.com/yandex/ClickHouse/pull/4009) ([alesapin](https://github.com/alesapin)) +* Исправлена утечка сетевых сокетов. Сокеты создавались в пуле и никогда не закрывались. При создании потока, создавались новые сокеты в случае, если все доступные использовались. [#4017](https://github.com/yandex/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn)) +* Исправлена проблема закрывания `/proc/self/fd` раньше, чем все файловые дескрипторы были прочитаны из `/proc` после создания процесса `odbc-bridge`. [#4120](https://github.com/yandex/ClickHouse/pull/4120) ([alesapin](https://github.com/alesapin)) +* Исправлен баг в монотонном преобразовании String в UInt в случае использования String в первичном ключе. [#3870](https://github.com/yandex/ClickHouse/pull/3870) ([Winter Zhang](https://github.com/zhang2014)) +* Исправлен баг в вычислении монотонности функции преобразования типа целых значений. [#3921](https://github.com/yandex/ClickHouse/pull/3921) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлено падение в функциях `arrayEnumerateUniq`, `arrayEnumerateDense` при передаче невалидных аргументов. [#3909](https://github.com/yandex/ClickHouse/pull/3909) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлен undefined behavior в StorageMerge. [#3910](https://github.com/yandex/ClickHouse/pull/3910) ([Amos Bird](https://github.com/amosbird)) +* Исправлено падение в функциях `addDays`, `subtractDays`. [#3913](https://github.com/yandex/ClickHouse/pull/3913) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлена проблема, в результате которой функции `round`, `floor`, `trunc`, `ceil` могли возвращать неверный результат для отрицательных целочисленных аргументов с большим значением. [#3914](https://github.com/yandex/ClickHouse/pull/3914) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлена проблема, в результате которой 'kill query sync' приводил к падению сервера. [#3916](https://github.com/yandex/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx)) +* Исправлен баг, приводящий к большой задержке в случае пустой очереди репликации. [#3928](https://github.com/yandex/ClickHouse/pull/3928) [#3932](https://github.com/yandex/ClickHouse/pull/3932) ([alesapin](https://github.com/alesapin)) +* Исправлено избыточное использование памяти в случае вставки в таблицу с `LowCardinality` в первичном ключе. [#3955](https://github.com/yandex/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Исправлена сериализация пустых массивов типа `LowCardinality` для формата `Native`. [#3907](https://github.com/yandex/ClickHouse/issues/3907) [#4011](https://github.com/yandex/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Исправлен неверный результат в случае использования distinct для числового столбца `LowCardinality`. [#3895](https://github.com/yandex/ClickHouse/issues/3895) [#4012](https://github.com/yandex/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Исправлена компиляция вычисления агрегатных функций для ключа `LowCardinality` (для случая, когда включена настройка `compile`). [#3886](https://github.com/yandex/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Исправлена передача пользователя и пароля для запросов с реплик. [#3957](https://github.com/yandex/ClickHouse/pull/3957) ([alesapin](https://github.com/alesapin)) ([小路](https://github.com/nicelulu)) +* Исправлен очень редкий race condition возникающий при перечислении таблиц из базы данных типа `Dictionary` во время перезагрузки словарей. [#3970](https://github.com/yandex/ClickHouse/pull/3970) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлен неверный результат в случае использования HAVING с ROLLUP или CUBE. [#3756](https://github.com/yandex/ClickHouse/issues/3756) [#3837](https://github.com/yandex/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection)) +* Исправлена проблема с алиасами столбцов для запросов с `JOIN ON` над распределенными таблицами. [#3980](https://github.com/yandex/ClickHouse/pull/3980) ([Winter Zhang](https://github.com/zhang2014)) +* Исправлена ошибка в реализации функции `quantileTDigest` (нашел Artem Vakhrushev). Эта ошибка никогда не происходит в ClickHouse и актуальна только для тех, кто использует кодовую базу ClickHouse напрямую в качестве библиотеки. [#3935](https://github.com/yandex/ClickHouse/pull/3935) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### Улучшения: + +* Добавлена поддержка `IF NOT EXISTS` в выражении `ALTER TABLE ADD COLUMN`, `IF EXISTS` в выражении `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [#3900](https://github.com/yandex/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea)) +* Функция `parseDateTimeBestEffort` теперь поддерживает форматы `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` и аналогичные. [#3922](https://github.com/yandex/ClickHouse/pull/3922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* `CapnProtoInputStream` теперь поддерживает jagged структуры. [#4063](https://github.com/yandex/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer)) +* Улучшение usability: добавлена проверка, что сервер запущен от пользователя, совпадающего с владельцем директории данных. Запрещен запуск от пользователя root в случае, если root не владеет директорией с данными. [#3785](https://github.com/yandex/ClickHouse/pull/3785) ([sergey-v-galtsev](https://github.com/sergey-v-galtsev)) +* Улучшена логика проверки столбцов, необходимых для JOIN, на стадии анализа запроса. [#3930](https://github.com/yandex/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2)) +* Уменьшено число поддерживаемых соединений в случае большого числа распределенных таблиц. [#3726](https://github.com/yandex/ClickHouse/pull/3726) ([Winter Zhang](https://github.com/zhang2014)) +* Добавлена поддержка строки с totals для запроса с `WITH TOTALS` через ODBC драйвер. [#3836](https://github.com/yandex/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb)) +* Поддержано использование `Enum` в качестве чисел в функции `if`. [#3875](https://github.com/yandex/ClickHouse/pull/3875) ([Ivan](https://github.com/abyss7)) +* Добавлена настройка `low_cardinality_allow_in_native_format`. Если она выключена, то тип `LowCadrinality` не используется в формате `Native`. [#3879](https://github.com/yandex/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai)) +* Удалены некоторые избыточные объекты из кеша скомпилированных выражений для уменьшения потребления памяти. [#4042](https://github.com/yandex/ClickHouse/pull/4042) ([alesapin](https://github.com/alesapin)) +* Добавлена проверка того, что в запрос `SET send_logs_level = 'value'` передается верное значение. [#3873](https://github.com/yandex/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx)) +* Добавлена проверка типов для функций преобразования типов. [#3896](https://github.com/yandex/ClickHouse/pull/3896) ([Winter Zhang](https://github.com/zhang2014)) + +### Улучшения производительности: + +* Добавлена настройка `use_minimalistic_part_header_in_zookeeper` для движка MergeTree. Если настройка включена, Replicated таблицы будут хранить метаданные куска в компактном виде (в соответствующем znode для этого куска). Это может значительно уменьшить размер для ZooKeeper snapshot (особенно для таблиц с большим числом столбцов). После включения данной настройки будет невозможно сделать откат к версии, которая эту настройку не поддерживает. [#3960](https://github.com/yandex/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn)) +* Добавлена реализация функций `sequenceMatch` и `sequenceCount` на основе конечного автомата в случае, если последовательность событий не содержит условия на время. [#4004](https://github.com/yandex/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo)) +* Улучшена производительность сериализации целых чисел. [#3968](https://github.com/yandex/ClickHouse/pull/3968) ([Amos Bird](https://github.com/amosbird)) +* Добавлен zero left padding для PODArray. Теперь элемент с индексом -1 является валидным нулевым значением. Эта особенность используется для удаления условного выражения при вычислении оффсетов массивов. [#3920](https://github.com/yandex/ClickHouse/pull/3920) ([Amos Bird](https://github.com/amosbird)) +* Откат версии `jemalloc`, приводящей к деградации производительности. [#4018](https://github.com/yandex/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov)) + +### Обратно несовместимые изменения: + +* Удалена недокументированная возможность `ALTER MODIFY PRIMARY KEY`, замененная выражением `ALTER MODIFY ORDER BY`. [#3887](https://github.com/yandex/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn)) +* Удалена функция `shardByHash`. [#3833](https://github.com/yandex/ClickHouse/pull/3833) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Запрещено использование скалярных подзапросов с результатом, имеющим тип `AggregateFunction`. [#3865](https://github.com/yandex/ClickHouse/pull/3865) ([Ivan](https://github.com/abyss7)) + +### Улучшения сборки/тестирования/пакетирования: + +* Добавлена поддержка сборки под PowerPC (`ppc64le`). [#4132](https://github.com/yandex/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1)) +* Функциональные stateful тесты запускаются на публично доступных данных. [#3969](https://github.com/yandex/ClickHouse/pull/3969) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлена ошибка, при которой сервер не мог запуститься с сообщением `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` при использовании Docker или systemd-nspawn. [#4136](https://github.com/yandex/ClickHouse/pull/4136) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Обновлена библиотека `rdkafka` до версии v1.0.0-RC5. Использована cppkafka на замену интерфейса языка C. [#4025](https://github.com/yandex/ClickHouse/pull/4025) ([Ivan](https://github.com/abyss7)) +* Обновлена библиотека `mariadb-client`. Исправлена проблема, обнаруженная с использованием UBSan. [#3924](https://github.com/yandex/ClickHouse/pull/3924) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправления для сборок с UBSan. [#3926](https://github.com/yandex/ClickHouse/pull/3926) [#3021](https://github.com/yandex/ClickHouse/pull/3021) [#3948](https://github.com/yandex/ClickHouse/pull/3948) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Добавлены покоммитные запуски тестов с UBSan сборкой. +* Добавлены покоммитные запуски тестов со статическим анализатором PVS-Studio. +* Исправлены проблемы, найденные с использованием PVS-Studio. [#4013](https://github.com/yandex/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправлены проблемы совместимости glibc. [#4100](https://github.com/yandex/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Docker образы перемещены на Ubuntu 18.10, добавлена совместимость с glibc >= 2.28 [#3965](https://github.com/yandex/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin)) +* Добавлена переменная окружения `CLICKHOUSE_DO_NOT_CHOWN`, позволяющая не делать shown директории для Docker образа сервера. [#3967](https://github.com/yandex/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin)) +* Включены большинство предупреждений из `-Weverything` для clang. Включено `-Wpedantic`. [#3986](https://github.com/yandex/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Добавлены некоторые предупреждения, специфичные только для clang 8. [#3993](https://github.com/yandex/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* При использовании динамической линковки используется `libLLVM` вместо библиотеки `LLVM`. [#3989](https://github.com/yandex/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej)) +* Добавлены переменные окружения для параметров `TSan`, `UBSan`, `ASan` в тестовом Docker образе. [#4072](https://github.com/yandex/ClickHouse/pull/4072) ([alesapin](https://github.com/alesapin)) +* Debian пакет `clickhouse-server` будет рекомендовать пакет `libcap2-bin` для того, чтобы использовать утилиту `setcap` для настроек. Данный пакет опционален. [#4093](https://github.com/yandex/ClickHouse/pull/4093) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Уменьшено время сборки, убраны ненужные включения заголовочных файлов. [#3898](https://github.com/yandex/ClickHouse/pull/3898) ([proller](https://github.com/proller)) +* Добавлены тесты производительности для функций хеширования. [#3918](https://github.com/yandex/ClickHouse/pull/3918) ([filimonov](https://github.com/filimonov)) +* Исправлены циклические зависимости библиотек. [#3958](https://github.com/yandex/ClickHouse/pull/3958) ([proller](https://github.com/proller)) +* Улучшена компиляция при малом объеме памяти. [#4030](https://github.com/yandex/ClickHouse/pull/4030) ([proller](https://github.com/proller)) +* Добавлен тестовый скрипт для воспроизведения деградации производительности в `jemalloc`. [#4036](https://github.com/yandex/ClickHouse/pull/4036) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Исправления опечаток в комментариях и строковых литералах. [#4122](https://github.com/yandex/ClickHouse/pull/4122) ([maiha](https://github.com/maiha)) +* Исправления опечаток в комментариях. [#4089](https://github.com/yandex/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty)) + ## ClickHouse release 18.16.1, 2018-12-21 ### Исправления ошибок: From 4b4a013827c386c48e09ffe6eb4af553776bc52b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Feb 2019 00:12:43 +0300 Subject: [PATCH 09/13] Update feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 60bd185195b..32f68fb987b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,6 +7,8 @@ assignees: '' --- +(you don't have to follow this form) + **Use case.** A clear and concise description of what is the intended usage scenario is. From 346adf896f4ad6a415e345fe8efa435ad1db5bca Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Feb 2019 00:12:54 +0300 Subject: [PATCH 10/13] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b62d1946711..f4ba6289e8c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,6 +7,8 @@ assignees: '' --- +(you don't have to follow this form) + **Describe the bug** A clear and concise description of what the bug is. From 1f95bf18e4a118637b6fb48abee213cdeca7fa1e Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 6 Feb 2019 11:46:41 +0300 Subject: [PATCH 11/13] Improve performance-test docker image --- docker/test/performance/Dockerfile | 15 ++++++++++++++ docker/test/performance/run.sh | 33 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 docker/test/performance/Dockerfile create mode 100755 docker/test/performance/run.sh diff --git a/docker/test/performance/Dockerfile b/docker/test/performance/Dockerfile new file mode 100644 index 00000000000..443f0d51060 --- /dev/null +++ b/docker/test/performance/Dockerfile @@ -0,0 +1,15 @@ +FROM yandex/clickhouse-stateful-test + +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + python-requests + +COPY s3downloader /s3downloader +COPY run.sh /run.sh + +ENV OPEN_DATASETS="hits" +ENV PRIVATE_DATASETS="hits_100m_single" +ENV DOWNLOAD_DATASETS=1 + +CMD /run.sh diff --git a/docker/test/performance/run.sh b/docker/test/performance/run.sh new file mode 100755 index 00000000000..2e149264ca0 --- /dev/null +++ b/docker/test/performance/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -x + +install_packages() { + dpkg -i package_folder/clickhouse-common-static_*.deb + dpkg -i package_folder/clickhouse-server_*.deb + dpkg -i package_folder/clickhouse-client_*.deb + dpkg -i package_folder/clickhouse-test_*.deb + service clickhouse-server start && sleep 5 +} + +download_data() { + clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" + clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" + /s3downloader --dataset-names $OPEN_DATASETS + /s3downloader --dataset-names $PRIVATE_DATASETS --url 'https://s3.mds.yandex.net/clickhouse-private-datasets' + chmod 777 -R /var/lib/clickhouse + service clickhouse-server restart && sleep 5 + clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" +} + +run() { + clickhouse-performance-test $TESTS_TO_RUN | tee test_output/test_result.json +} + +install_packages + +if [ $DOWNLOAD_DATASETS -eq 1 ]; then + download_data +fi + +run From 321caa7f5c813306b0f7ba108008ee641e13a868 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 6 Feb 2019 14:44:00 +0300 Subject: [PATCH 12/13] Enable all metrics in performance test by default --- .../performance-test/PerformanceTestInfo.cpp | 76 ++----------- .../performance-test/PerformanceTestInfo.h | 1 - .../performance-test/ReportBuilder.cpp | 100 ++++++++---------- 3 files changed, 55 insertions(+), 122 deletions(-) diff --git a/dbms/programs/performance-test/PerformanceTestInfo.cpp b/dbms/programs/performance-test/PerformanceTestInfo.cpp index e10fd1e915f..cfb9ae0fd92 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.cpp +++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp @@ -36,42 +36,6 @@ void extractSettings( } } -void checkMetricsInput(const Strings & metrics, ExecutionType exec_type) -{ - Strings loop_metrics = { - "min_time", "quantiles", "total_time", - "queries_per_second", "rows_per_second", - "bytes_per_second"}; - - Strings non_loop_metrics = { - "max_rows_per_second", "max_bytes_per_second", - "avg_rows_per_second", "avg_bytes_per_second"}; - - if (exec_type == ExecutionType::Loop) - { - for (const std::string & metric : metrics) - { - auto non_loop_pos = - std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric); - - if (non_loop_pos != non_loop_metrics.end()) - throw Exception("Wrong type of metric for loop execution type (" + metric + ")", - ErrorCodes::BAD_ARGUMENTS); - } - } - else - { - for (const std::string & metric : metrics) - { - auto loop_pos = std::find(loop_metrics.begin(), loop_metrics.end(), metric); - if (loop_pos != loop_metrics.end()) - throw Exception( - "Wrong type of metric for non-loop execution type (" + metric + ")", - ErrorCodes::BAD_ARGUMENTS); - } - } -} - } @@ -84,12 +48,19 @@ PerformanceTestInfo::PerformanceTestInfo( { test_name = config->getString("name"); path = config->getString("path"); + if (config->has("main_metric")) + { + Strings main_metrics; + config->keys("main_metric", main_metrics); + if (main_metrics.size()) + main_metric = main_metrics[0]; + } + applySettings(config); extractQueries(config); processSubstitutions(config); getExecutionType(config); getStopConditions(config); - getMetrics(config); extractAuxiliaryQueries(config); } @@ -239,37 +210,6 @@ void PerformanceTestInfo::getStopConditions(XMLConfigurationPtr config) } - -void PerformanceTestInfo::getMetrics(XMLConfigurationPtr config) -{ - ConfigurationPtr metrics_view(config->createView("metrics")); - metrics_view->keys(metrics); - - if (config->has("main_metric")) - { - Strings main_metrics; - config->keys("main_metric", main_metrics); - if (main_metrics.size()) - main_metric = main_metrics[0]; - } - - if (!main_metric.empty()) - { - if (std::find(metrics.begin(), metrics.end(), main_metric) == metrics.end()) - metrics.push_back(main_metric); - } - else - { - if (metrics.empty()) - throw Exception("You shoud specify at least one metric", - ErrorCodes::BAD_ARGUMENTS); - main_metric = metrics[0]; - } - - if (metrics.size() > 0) - checkMetricsInput(metrics, exec_type); -} - void PerformanceTestInfo::extractAuxiliaryQueries(XMLConfigurationPtr config) { if (config->has("create_query")) diff --git a/dbms/programs/performance-test/PerformanceTestInfo.h b/dbms/programs/performance-test/PerformanceTestInfo.h index 9b84a885de0..e807f543dde 100644 --- a/dbms/programs/performance-test/PerformanceTestInfo.h +++ b/dbms/programs/performance-test/PerformanceTestInfo.h @@ -33,7 +33,6 @@ public: std::string main_metric; Strings queries; - Strings metrics; Settings settings; ExecutionType exec_type; diff --git a/dbms/programs/performance-test/ReportBuilder.cpp b/dbms/programs/performance-test/ReportBuilder.cpp index 31572270d31..a74721a7faf 100644 --- a/dbms/programs/performance-test/ReportBuilder.cpp +++ b/dbms/programs/performance-test/ReportBuilder.cpp @@ -17,6 +17,18 @@ namespace DB namespace { const std::regex QUOTE_REGEX{"\""}; +std::string getMainMetric(const PerformanceTestInfo & test_info) +{ + std::string main_metric; + if (test_info.main_metric.empty()) + if (test_info.exec_type == ExecutionType::Loop) + main_metric = "min_time"; + else + main_metric = "rows_per_second"; + else + main_metric = test_info.main_metric; + return main_metric; +} } ReportBuilder::ReportBuilder(const std::string & server_version_) @@ -48,13 +60,7 @@ std::string ReportBuilder::buildFullReport( json_output.set("time", getCurrentTime()); json_output.set("test_name", test_info.test_name); json_output.set("path", test_info.path); - json_output.set("main_metric", test_info.main_metric); - - auto has_metric = [&test_info] (const std::string & metric_name) - { - return std::find(test_info.metrics.begin(), - test_info.metrics.end(), metric_name) != test_info.metrics.end(); - }; + json_output.set("main_metric", getMainMetric(test_info)); if (test_info.substitutions.size()) { @@ -108,61 +114,46 @@ std::string ReportBuilder::buildFullReport( if (test_info.exec_type == ExecutionType::Loop) { /// in seconds - if (has_metric("min_time")) - runJSON.set("min_time", statistics.min_time / double(1000)); + runJSON.set("min_time", statistics.min_time / double(1000)); - if (has_metric("quantiles")) + JSONString quantiles(4); /// here, 4 is the size of \t padding + for (double percent = 10; percent <= 90; percent += 10) { - JSONString quantiles(4); /// here, 4 is the size of \t padding - for (double percent = 10; percent <= 90; percent += 10) - { - std::string quantile_key = std::to_string(percent / 100.0); - while (quantile_key.back() == '0') - quantile_key.pop_back(); + std::string quantile_key = std::to_string(percent / 100.0); + while (quantile_key.back() == '0') + quantile_key.pop_back(); - quantiles.set(quantile_key, - statistics.sampler.quantileInterpolated(percent / 100.0)); - } - quantiles.set("0.95", - statistics.sampler.quantileInterpolated(95 / 100.0)); - quantiles.set("0.99", - statistics.sampler.quantileInterpolated(99 / 100.0)); - quantiles.set("0.999", - statistics.sampler.quantileInterpolated(99.9 / 100.0)); - quantiles.set("0.9999", - statistics.sampler.quantileInterpolated(99.99 / 100.0)); - - runJSON.set("quantiles", quantiles.asString()); + quantiles.set(quantile_key, + statistics.sampler.quantileInterpolated(percent / 100.0)); } + quantiles.set("0.95", + statistics.sampler.quantileInterpolated(95 / 100.0)); + quantiles.set("0.99", + statistics.sampler.quantileInterpolated(99 / 100.0)); + quantiles.set("0.999", + statistics.sampler.quantileInterpolated(99.9 / 100.0)); + quantiles.set("0.9999", + statistics.sampler.quantileInterpolated(99.99 / 100.0)); - if (has_metric("total_time")) - runJSON.set("total_time", statistics.total_time); + runJSON.set("quantiles", quantiles.asString()); - if (has_metric("queries_per_second")) - runJSON.set("queries_per_second", - double(statistics.queries) / statistics.total_time); + runJSON.set("total_time", statistics.total_time); - if (has_metric("rows_per_second")) - runJSON.set("rows_per_second", - double(statistics.total_rows_read) / statistics.total_time); + runJSON.set("queries_per_second", + static_cast(statistics.queries) / statistics.total_time); - if (has_metric("bytes_per_second")) - runJSON.set("bytes_per_second", - double(statistics.total_bytes_read) / statistics.total_time); + runJSON.set("rows_per_second", + static_cast(statistics.total_rows_read) / statistics.total_time); + + runJSON.set("bytes_per_second", + static_cast(statistics.total_bytes_read) / statistics.total_time); } else { - if (has_metric("max_rows_per_second")) - runJSON.set("max_rows_per_second", statistics.max_rows_speed); - - if (has_metric("max_bytes_per_second")) - runJSON.set("max_bytes_per_second", statistics.max_bytes_speed); - - if (has_metric("avg_rows_per_second")) - runJSON.set("avg_rows_per_second", statistics.avg_rows_speed_value); - - if (has_metric("avg_bytes_per_second")) - runJSON.set("avg_bytes_per_second", statistics.avg_bytes_speed_value); + runJSON.set("max_rows_per_second", statistics.max_rows_speed); + runJSON.set("max_bytes_per_second", statistics.max_bytes_speed); + runJSON.set("avg_rows_per_second", statistics.avg_rows_speed_value); + runJSON.set("avg_bytes_per_second", statistics.avg_bytes_speed_value); } run_infos.push_back(runJSON); @@ -193,9 +184,12 @@ std::string ReportBuilder::buildCompactReport( output << "query \"" << test_info.queries[query_index] << "\", "; output << "run " << std::to_string(number_of_launch + 1) << ": "; - output << test_info.main_metric << " = "; + + std::string main_metric = getMainMetric(test_info); + + output << main_metric << " = "; size_t index = number_of_launch * test_info.queries.size() + query_index; - output << stats[index].getStatisticByName(test_info.main_metric); + output << stats[index].getStatisticByName(main_metric); output << "\n"; } } From 54d70e5f4c3a92f9f544a90b30b0e08b132a664f Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 6 Feb 2019 15:52:46 +0300 Subject: [PATCH 13/13] DOCAPI-4994: New article "ClickHouse Update" (#4285) --- docs/en/operations/index.md | 3 ++- docs/en/operations/update.md | 13 +++++++++++++ docs/fa/operations/update.md | 1 + docs/ru/operations/index.md | 1 + docs/ru/operations/update.md | 13 +++++++++++++ docs/toc_en.yml | 1 + docs/toc_fa.yml | 3 ++- docs/toc_ru.yml | 1 + docs/toc_zh.yml | 1 + docs/zh/operations/update.md | 1 + 10 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docs/en/operations/update.md create mode 120000 docs/fa/operations/update.md create mode 100644 docs/ru/operations/update.md create mode 120000 docs/zh/operations/update.md diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md index 399a180ec46..b864f327a2e 100644 --- a/docs/en/operations/index.md +++ b/docs/en/operations/index.md @@ -1,4 +1,4 @@ -# Operations +# Operations ClickHouse operations manual consists of the following major sections: @@ -6,6 +6,7 @@ ClickHouse operations manual consists of the following major sections: - [Monitoring](monitoring.md) - [Troubleshooting](troubleshooting.md) - [Usage Recommendations](tips.md) + - [ClickHouse Update](update.md) - [Access Rights](access_rights.md) - [Data Backup](backup.md) - [Configuration Files](configuration_files.md) diff --git a/docs/en/operations/update.md b/docs/en/operations/update.md new file mode 100644 index 00000000000..067dff76f4a --- /dev/null +++ b/docs/en/operations/update.md @@ -0,0 +1,13 @@ +# ClickHouse Update + +If ClickHouse is installed from deb-packages, execute the following commands on the server: + +``` +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +sudo service clickhouse-server restart +``` + +If you installed ClickHouse not from recommended deb-packages, use corresponding methods of update. + +ClickHouse does not support a distributed update. The operation should be performed consecutively at each separate server. Do not update all the servers on cluster at the same time, otherwise cluster became unavailable for some time. diff --git a/docs/fa/operations/update.md b/docs/fa/operations/update.md new file mode 120000 index 00000000000..88a092c0dff --- /dev/null +++ b/docs/fa/operations/update.md @@ -0,0 +1 @@ +../../en/operations/update.md \ No newline at end of file diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 5e691e81047..9ad2fc5ff76 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -6,6 +6,7 @@ - [Мониторинг](monitoring.md) - [Решение проблем](troubleshooting.md) - [Советы по эксплуатации](tips.md) + - [Обновление ClickHouse](update.md) - [Права доступа](access_rights.md) - [Резервное копирование](backup.md) - [Конфигурационные файлы](configuration_files.md) diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md new file mode 100644 index 00000000000..41346775637 --- /dev/null +++ b/docs/ru/operations/update.md @@ -0,0 +1,13 @@ +# Обновление ClickHouse + +Если ClickHouse установлен с помощью deb-пакетов, выполните следующие команды на сервере: + +``` +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +sudo service clickhouse-server restart +``` + +Если ClickHouse установлен не из рекомендуемых deb-пакетов, используйте соответствующий метод обновления. + +ClickHouse не поддерживает распределенное обновление. Операция должна выполняться последовательно на каждом отдельном сервере. Не обновляйте все серверы в кластере одновременно, иначе кластер становится недоступным в течение некоторого времени. diff --git a/docs/toc_en.yml b/docs/toc_en.yml index f41e94b1a56..d27929eb855 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -162,6 +162,7 @@ nav: - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' - 'Usage Recommendations': 'operations/tips.md' + - 'ClickHouse Update': 'operations/update.md' - 'Access Rights': 'operations/access_rights.md' - 'Data Backup': 'operations/backup.md' - 'Configuration Files': 'operations/configuration_files.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 16a4b2f729f..f51c11125c8 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -17,7 +17,7 @@ nav: - ' ترابایت از لاگ های کلیک از سرویس Criteo': 'getting_started/example_datasets/criteo.md' - ' بنچمارک Star Schema': 'getting_started/example_datasets/star_schema.md' - 'Yandex.Metrica Data': 'getting_started/example_datasets/metrica.md' - + - 'Interfaces': - 'Interface ها': 'interfaces/index.md' - ' کلاینت Command-line': 'interfaces/cli.md' @@ -162,6 +162,7 @@ nav: - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' - 'Usage Recommendations': 'operations/tips.md' + - 'ClickHouse Update': 'operations/update.md' - 'Access Rights': 'operations/access_rights.md' - 'Data Backup': 'operations/backup.md' - 'Configuration Files': 'operations/configuration_files.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 3b588f37896..60a4df97e0d 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -160,6 +160,7 @@ nav: - 'Мониторинг': 'operations/monitoring.md' - 'Решение проблем': 'operations/troubleshooting.md' - 'Советы по эксплуатации': 'operations/tips.md' + - 'Обновление ClickHouse': 'operations/update.md' - 'Права доступа': 'operations/access_rights.md' - 'Резервное копирование': 'operations/backup.md' - 'Конфигурационные файлы': 'operations/configuration_files.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index ed4f3da3fe2..30fc4a2fa66 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -161,6 +161,7 @@ nav: - 'Monitoring': 'operations/monitoring.md' - 'Troubleshooting': 'operations/troubleshooting.md' - 'Usage recommendations': 'operations/tips.md' + - 'ClickHouse Update': 'operations/update.md' - 'Access rights': 'operations/access_rights.md' - 'Data backup': 'operations/backup.md' - 'Configuration files': 'operations/configuration_files.md' diff --git a/docs/zh/operations/update.md b/docs/zh/operations/update.md new file mode 120000 index 00000000000..88a092c0dff --- /dev/null +++ b/docs/zh/operations/update.md @@ -0,0 +1 @@ +../../en/operations/update.md \ No newline at end of file