From 1d1070af7165c14dd08181b9cbaff51862b719ce Mon Sep 17 00:00:00 2001 From: Gagan Arneja Date: Sun, 29 Mar 2020 18:12:29 -0700 Subject: [PATCH 01/33] Fix a race between DDLWorker and DDLWorkerClnr during startup that causes DDLWorker to exit because of ZooKeeper NONODE error --- dbms/src/Interpreters/DDLWorker.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 8fe87eb9f3b..eaee356264d 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -421,6 +421,18 @@ void DDLWorker::processTasks() { processTask(task, zookeeper); } + catch (const Coordination::Exception & e) + { + if (server_startup && e.code == Coordination::ZNONODE) + { + LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << + task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + } + else + { + throw; + } + } catch (...) { LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " From 8d19f9f7d7ee9f1c770b0afbbe3288842c768afb Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 30 Mar 2020 13:21:08 +0300 Subject: [PATCH 02/33] Add hard timeouts to alter tests --- .../01079_parallel_alter_add_drop_column_zookeeper.sh | 2 +- .../0_stateless/01079_parallel_alter_detach_table_zookeeper.sh | 2 +- .../0_stateless/01079_parallel_alter_modify_zookeeper.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index 1fbccf47c72..37ed463f59b 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -97,7 +97,7 @@ done echo "Equal number of columns" # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 0ec6f01137f..114008ded26 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -96,7 +96,7 @@ done # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 5d181727301..bacc742d16a 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -101,7 +101,7 @@ wait echo "Finishing alters" # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done From d8ff25d3647f63b3690dd4ae9918ebbffe458635 Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Mon, 30 Mar 2020 19:29:13 +0900 Subject: [PATCH 03/33] Fix a bug that windowFunnel() returns null. --- .../AggregateFunctionNull.cpp | 8 ++++++-- .../AggregateFunctionNull.h | 8 ++++---- .../AggregateFunctionWindowFunnel.h | 7 ++++++- .../AggregateFunctions/IAggregateFunction.h | 9 +++++++-- .../00632_aggregation_window_funnel.reference | 20 +++++++++++++++++++ .../00632_aggregation_window_funnel.sql | 13 ++++++++++++ 6 files changed, 56 insertions(+), 9 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp b/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp index e577df472c8..60712636562 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -51,6 +51,10 @@ public: if (!has_nullable_types) throw Exception("Aggregate function combinator 'Null' requires at least one argument to be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (nested_function) + if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params)) + return adapter; + /// Special case for 'count' function. It could be called with Nullable arguments /// - that means - count number of calls, when all arguments are not NULL. if (nested_function && nested_function->getName() == "count") @@ -71,9 +75,9 @@ public: else { if (return_type_is_nullable) - return std::make_shared>(nested_function, arguments, params); + return std::make_shared>(nested_function, arguments, params); else - return std::make_shared>(nested_function, arguments, params); + return std::make_shared>(nested_function, arguments, params); } } }; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionNull.h b/dbms/src/AggregateFunctions/AggregateFunctionNull.h index a5000f30cd5..a0fe96b6f62 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionNull.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionNull.h @@ -204,12 +204,12 @@ public: }; -template -class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase> +template +class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase> { public: AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) - : AggregateFunctionNullBase>(std::move(nested_function_), arguments, params), + : AggregateFunctionNullBase>(std::move(nested_function_), arguments, params), number_of_arguments(arguments.size()) { if (number_of_arguments == 1) @@ -233,7 +233,7 @@ public: if (is_nullable[i]) { const ColumnNullable & nullable_col = assert_cast(*columns[i]); - if (nullable_col.isNullAt(row_num)) + if (null_is_skipped && nullable_col.isNullAt(row_num)) { /// If at least one column has a null value in the current row, /// we don't process this row. diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index e8668b6172e..e19751d8daa 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -11,7 +11,7 @@ #include #include -#include +#include namespace DB { @@ -232,6 +232,11 @@ public: return std::make_shared(); } + AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override + { + return std::make_shared>(nested_function, arguments, params); + } + void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override { bool has_event = false; diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index d7ccd4c206a..5cf8d90aaa3 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -31,6 +31,8 @@ using DataTypes = std::vector; using AggregateDataPtr = char *; using ConstAggregateDataPtr = const char *; +class IAggregateFunction; +using AggregateFunctionPtr = std::shared_ptr; /** Aggregate functions interface. * Instances of classes with this interface do not contain the data itself for aggregation, @@ -149,6 +151,11 @@ public: virtual void addBatchArray( size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena) const = 0; + virtual AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & /*nested_function*/, const DataTypes & /*arguments*/, const Array & /*params*/) const + { + return nullptr; + } + const DataTypes & getArgumentTypes() const { return argument_types; } const Array & getParameters() const { return parameters; } @@ -244,6 +251,4 @@ public: }; -using AggregateFunctionPtr = std::shared_ptr; - } diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference index 1e572be797c..04851cc0e83 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference @@ -36,3 +36,23 @@ [4, 2] [5, 2] [6, 1] +[1, 2] +[2, 2] +[3, 0] +[4, 0] +[1, 2] +[2, 1] +[3, 0] +[4, 0] +[1, 0] +[2, 0] +[3, 1] +[4, 0] +[1, 0] +[2, 0] +[3, 1] +[4, 2] +[1, 0] +[2, 0] +[3, 1] +[4, 1] diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql index a8a2f522be9..43ae0197782 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql @@ -64,3 +64,16 @@ select user, windowFunnel(86400)(dt, event='a', event='b', event='c') as s from select user, windowFunnel(86400, 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; select user, windowFunnel(86400, 'strict', 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; drop table funnel_test_strict_order; + +drop table if exists funnel_test_non_null; +create table funnel_test_non_null (`dt` DateTime, `u` int, `a` Nullable(String), `b` Nullable(String)) engine = MergeTree() partition by dt order by u; +insert into funnel_test_non_null values (1, 1, 'a1', 'b1') (2, 1, 'a2', 'b2'); +insert into funnel_test_non_null values (1, 2, 'a1', null) (2, 2, 'a2', null); +insert into funnel_test_non_null values (1, 3, null, null); +insert into funnel_test_non_null values (1, 4, null, 'b1') (2, 4, 'a2', null) (3, 4, null, 'b3'); +select u, windowFunnel(86400)(dt, a = 'a1', a = 'a2') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a = 'a1', b = 'b2') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a is null and b is null) as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400, 'strict_order')(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +drop table funnel_test_non_null; From 210024cd3a878a55d0666446f45ae952fb74abc9 Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Mon, 30 Mar 2020 19:35:54 +0900 Subject: [PATCH 04/33] Fix a corner case of windowFunnel with 'strict_order' --- .../AggregateFunctions/AggregateFunctionWindowFunnel.h | 8 ++++++++ .../0_stateless/00632_aggregation_window_funnel.reference | 1 + .../0_stateless/00632_aggregation_window_funnel.sql | 2 ++ 3 files changed, 11 insertions(+) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index e19751d8daa..3a1d2adee4a 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -186,6 +186,14 @@ private: { return event_idx + 1; } + else if (strict_order && first_event && events_timestamp[event_idx - 1] == -1) + { + for (size_t event = 0; event < events_timestamp.size(); ++event) + { + if (events_timestamp[event] == -1) + return event; + } + } else if (events_timestamp[event_idx - 1] >= 0 && timestamp <= events_timestamp[event_idx - 1] + window) { events_timestamp[event_idx] = events_timestamp[event_idx - 1]; diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference index 04851cc0e83..492135567ea 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference @@ -36,6 +36,7 @@ [4, 2] [5, 2] [6, 1] +[7, 1] [1, 2] [2, 2] [3, 0] diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql index 43ae0197782..5a1610256ac 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql @@ -63,6 +63,8 @@ insert into funnel_test_strict_order values (1, 6, 'c') (2, 6, 'c') (3, 6, 'b') select user, windowFunnel(86400)(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; select user, windowFunnel(86400, 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; select user, windowFunnel(86400, 'strict', 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; +insert into funnel_test_strict_order values (1, 7, 'a') (2, 7, 'c') (3, 7, 'b'); +select user, windowFunnel(10, 'strict_order')(dt, event = 'a', event = 'b', event = 'c') as s from funnel_test_strict_order where user = 7 group by user format JSONCompactEachRow; drop table funnel_test_strict_order; drop table if exists funnel_test_non_null; From 19c3939b62f7630acd575fab97c48f5884022657 Mon Sep 17 00:00:00 2001 From: AsiaKorushkina <43650329+AsiaKorushkina@users.noreply.github.com> Date: Mon, 30 Mar 2020 15:23:54 +0300 Subject: [PATCH 05/33] asiana21 DOCSUP-658 (#93) * docs(regionToTopContinent): part of the function description * docs(regionToTopContinent): some shanges are added in the description * Update docs/en/query_language/functions/ym_dict_functions.md Co-Authored-By: BayoNet * Update docs/en/query_language/functions/ym_dict_functions.md Co-Authored-By: BayoNet * Update docs/en/query_language/functions/ym_dict_functions.md Co-Authored-By: BayoNet * Update docs/en/query_language/functions/ym_dict_functions.md Co-Authored-By: BayoNet * docs(regionToTopContinent): some shanges * docs(regionToTopContinent): added translation of the function description Co-authored-by: asiana21 Co-authored-by: BayoNet --- .../functions/ym_dict_functions.md | 26 ++++++++++++++++++- .../functions/ym_dict_functions.md | 24 ++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/en/query_language/functions/ym_dict_functions.md b/docs/en/query_language/functions/ym_dict_functions.md index e83e86d73f9..5dfcaa54fe3 100644 --- a/docs/en/query_language/functions/ym_dict_functions.md +++ b/docs/en/query_language/functions/ym_dict_functions.md @@ -4,7 +4,7 @@ In order for the functions below to work, the server config must specify the pat For information about creating reference lists, see the section "Dictionaries". -## Multiple geobases +## Multiple Geobases {#multiplegeobases} ClickHouse supports working with multiple alternative geobases (regional hierarchies) simultaneously, in order to support various perspectives on which countries certain regions belong to. @@ -100,6 +100,30 @@ Example: `regionToCountry(toUInt32(213)) = 225` converts Moscow (213) to Russia Converts a region to a continent. In every other way, this function is the same as 'regionToCity'. Example: `regionToContinent(toUInt32(213)) = 10001` converts Moscow (213) to Eurasia (10001). +### regionToTopContinent (#regiontotopcontinent) + +Finds the highest continent in the hierarchy for the region. + +**Syntax** + +```sql +regionToTopContinent(id[, geobase]); +``` + +**Parameters** + +- `id` — Region ID from the Yandex geobase. [UInt32](../../data_types/int_uint.md). +- `geobase` — Dictionary key. See [Multiple Geobases](#multiplegeobases). [String](../../data_types/string.md). Optional. + + +**Returned value** + +- Identifier of the top level continent (the latter when you climb the hierarchy of regions). +- 0, if there is none. + +Type: `UInt32`. + + ### regionToPopulation(id[, geobase]) Gets the population for a region. diff --git a/docs/ru/query_language/functions/ym_dict_functions.md b/docs/ru/query_language/functions/ym_dict_functions.md index 4340d6c8e16..f4abb2043c4 100644 --- a/docs/ru/query_language/functions/ym_dict_functions.md +++ b/docs/ru/query_language/functions/ym_dict_functions.md @@ -4,7 +4,7 @@ О том, как создать справочники, смотрите в разделе "Словари". -## Множественные геобазы +## Множественные геобазы {#multiplegeobases} ClickHouse поддерживает работу одновременно с несколькими альтернативными геобазами (иерархиями регионов), для того чтобы можно было поддержать разные точки зрения о принадлежности регионов странам. @@ -96,6 +96,28 @@ LIMIT 15 Переводит регион в континент. В остальном, аналогично функции regionToCity. Пример: `regionToContinent(toUInt32(213)) = 10001` - преобразовали Москву (213) в Евразию (10001). +### regionToTopContinent (#regiontotopcontinent) + +Находит для региона верхний в иерархии континент. + +**Синтаксис** + +```sql +regionToTopContinent(id[, geobase]); +``` + +**Параметры** + +- `id` — Идентификатор региона из геобазы Яндекса. [UInt32](../../data_types/int_uint.md). +- `geobase` — Ключ словаря. Смотрите [Множественные геобазы](#multiplegeobases). [String](../../data_types/string.md). Опциональный параметр. + +**Возвращаемое значение** + +- Идентификатор верхнего уровня континента (последний при подъеме по иерархии регионов). +- 0, если его нет. + +Тип: `UInt32`. + ### regionToPopulation(id\[, geobase\]) Получает население для региона. Население может быть прописано в файлах с геобазой. Смотрите в разделе "Встроенные словари". From 14d078d7fb68c9f099ff5f1451c6efcdaf497096 Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Mon, 30 Mar 2020 15:25:54 +0300 Subject: [PATCH 06/33] CLICKHOUSEDOCS-138: Fixed lang. --- docs/ru/query_language/functions/ym_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/query_language/functions/ym_dict_functions.md b/docs/ru/query_language/functions/ym_dict_functions.md index f4abb2043c4..22502f26dfc 100644 --- a/docs/ru/query_language/functions/ym_dict_functions.md +++ b/docs/ru/query_language/functions/ym_dict_functions.md @@ -113,7 +113,7 @@ regionToTopContinent(id[, geobase]); **Возвращаемое значение** -- Идентификатор верхнего уровня континента (последний при подъеме по иерархии регионов). +- Идентификатор континента верхнего уровня (последний при подъеме по иерархии регионов). - 0, если его нет. Тип: `UInt32`. From 7c2c8e2d1ba7e7a7822ee66a533258f6df9c492f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 30 Mar 2020 22:13:03 +0300 Subject: [PATCH 07/33] Update security_changelog.md --- docs/ru/security_changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ru/security_changelog.md b/docs/ru/security_changelog.md index d0becdbde9d..c8f66bf0475 100644 --- a/docs/ru/security_changelog.md +++ b/docs/ru/security_changelog.md @@ -65,4 +65,5 @@ unixODBC позволял указать путь для подключения Некорректная конфигурация в deb пакете могла привести к неавторизованному доступу к базе данных. Обнаружено благодаря: the UK’s National Cyber Security Centre (NCSC) -[Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) + +{## [Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) ##} From 2f4033539015a2ef05a1571c3d1ed47391c78023 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 30 Mar 2020 22:33:33 +0300 Subject: [PATCH 08/33] Extra logging at build.py + rearrange ifs --- docs/tools/build.py | 52 +++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 741e9343923..2e764f1c5b0 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -218,39 +218,39 @@ def build_single_page_version(lang, args, cfg): os.path.join(site_temp, 'single'), single_page_output_path ) + + logging.info(f'Re-building single page for {lang} pdf/test') + with util.temp_dir() as test_dir: + single_page_pdf = os.path.abspath( + os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf') + ) + extra['single_page'] = False + cfg.load_dict({ + 'docs_dir': docs_temp_lang, + 'site_dir': test_dir, + 'extra': extra, + 'nav': [ + {cfg.data.get('site_name'): 'single.md'} + ] + }) + mkdocs_build.build(cfg) - if not args.skip_pdf: - with util.temp_dir() as test_dir: - single_page_pdf = os.path.abspath( - os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf') - ) - extra['single_page'] = False - cfg.load_dict({ - 'docs_dir': docs_temp_lang, - 'site_dir': test_dir, - 'extra': extra, - 'nav': [ - {cfg.data.get('site_name'): 'single.md'} - ] - }) - mkdocs_build.build(cfg) - - css_in = ' '.join(website.get_css_in(args)) - js_in = ' '.join(website.get_js_in(args)) - subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) - subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + css_in = ' '.join(website.get_css_in(args)) + js_in = ' '.join(website.get_js_in(args)) + subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) + subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + if not args.skip_pdf: port_for_pdf = util.get_free_port() with socketserver.TCPServer( ('', port_for_pdf), http.server.SimpleHTTPRequestHandler ) as httpd: - logging.info(f"serving for pdf at port {port_for_pdf}") + logging.info(f"Serving for {lang} pdf at port {port_for_pdf}") thread = threading.Thread(target=httpd.serve_forever) with util.cd(test_dir): thread.start() create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', - '--no-stop-slow-scripts', '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] @@ -262,10 +262,12 @@ def build_single_page_version(lang, args, cfg): finally: httpd.shutdown() thread.join(timeout=5.0) + logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}") - if not args.version_prefix: # maybe enable in future - test.test_single_page( - os.path.join(test_dir, 'single', 'index.html'), lang) + if not args.version_prefix: # maybe enable in future + logging.info(f'Running tests for {lang}') + test.test_single_page( + os.path.join(test_dir, 'single', 'index.html'), lang) logging.info(f'Finished building single page version for {lang}') From 4c8c2d2933886a859a28c9fc1f0b6cd9b5569ccc Mon Sep 17 00:00:00 2001 From: "philip.han" Date: Tue, 31 Mar 2020 06:52:16 +0900 Subject: [PATCH 09/33] Add a comment of getOwnNullAdapter() --- dbms/src/AggregateFunctions/IAggregateFunction.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index 5cf8d90aaa3..48d31793a5a 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -151,6 +151,12 @@ public: virtual void addBatchArray( size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena) const = 0; + /** By default all NULLs are skipped during aggregation. + * If it returns nullptr, the default one will be used. + * If an aggregate function wants to use something instead of the default one, it overrides this function and returns its own null adapter. + * nested_function is a smart pointer to this aggregate function itself. + * arguments and params are for nested_function. + */ virtual AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & /*nested_function*/, const DataTypes & /*arguments*/, const Array & /*params*/) const { return nullptr; From 1a7cdfb18b0d1a8828987a79dfe9d91d7891f67f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 10:01:12 +0300 Subject: [PATCH 10/33] Try to get rid from httpd context manager (#9952) * Try to get rid from httpd context manager * Update build.py --- docs/tools/build.py | 46 ++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 2e764f1c5b0..167936a7fc7 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -6,12 +6,12 @@ import argparse import datetime import http.server import logging +import multiprocessing import os import shutil import socketserver import subprocess import sys -import threading import time import jinja2 @@ -241,28 +241,28 @@ def build_single_page_version(lang, args, cfg): subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) if not args.skip_pdf: port_for_pdf = util.get_free_port() - with socketserver.TCPServer( - ('', port_for_pdf), http.server.SimpleHTTPRequestHandler - ) as httpd: - logging.info(f"Serving for {lang} pdf at port {port_for_pdf}") - thread = threading.Thread(target=httpd.serve_forever) - with util.cd(test_dir): - thread.start() - create_pdf_command = [ - 'wkhtmltopdf', - '--print-media-type', - '--log-level', 'warn', - f'http://localhost:{port_for_pdf}/single/', single_page_pdf - ] - try: - if args.save_raw_single_page: - shutil.copytree(test_dir, args.save_raw_single_page) - logging.info(' '.join(create_pdf_command)) - subprocess.check_call(' '.join(create_pdf_command), shell=True) - finally: - httpd.shutdown() - thread.join(timeout=5.0) - logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}") + httpd = socketserver.TCPServer( + ('', port_for_pdf), http.server.SimpleHTTPRequestHandler + ) + logging.info(f"Serving for {lang} pdf at port {port_for_pdf}") + process = multiprocessing.Process(target=httpd.serve_forever) + with util.cd(test_dir): + thread.start() + create_pdf_command = [ + 'wkhtmltopdf', + '--print-media-type', + '--log-level', 'warn', + f'http://localhost:{port_for_pdf}/single/', single_page_pdf + ] + try: + if args.save_raw_single_page: + shutil.copytree(test_dir, args.save_raw_single_page) + logging.info(' '.join(create_pdf_command)) + subprocess.check_call(' '.join(create_pdf_command), shell=True) + finally: + logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}") + process.terminate() + if not args.version_prefix: # maybe enable in future logging.info(f'Running tests for {lang}') From 23c34f8b5efd7c4b2d7ec3b153d763f7b7b4013f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 10:18:18 +0300 Subject: [PATCH 11/33] fix mistype --- docs/tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 167936a7fc7..de94255c9f5 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -247,7 +247,7 @@ def build_single_page_version(lang, args, cfg): logging.info(f"Serving for {lang} pdf at port {port_for_pdf}") process = multiprocessing.Process(target=httpd.serve_forever) with util.cd(test_dir): - thread.start() + process.start() create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', From 747bc1451429bdc39229384759361bf9994c37e0 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 11:01:56 +0300 Subject: [PATCH 12/33] Update build.py --- docs/tools/build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index de94255c9f5..42ea30220d0 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import argparse import datetime import http.server From ab701ec0c5159f614e121628b368fcca13a3747f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Mar 2020 11:54:46 +0300 Subject: [PATCH 13/33] Fix 00975_indices_mutation_replicated_zookeeper test --- .../00975_indices_mutation_replicated_zookeeper.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index c6e16fc5148..28e7bd0b94e 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -46,15 +46,12 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" -sleep 1 +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "indices_mutaions1" "0000000000" "test" -wait_for_mutation "indices_mutaions2" "0000000000" "test" +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" From 0b2bcd5bcd6e1fda186920438ece23d4b7a411c1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 12:35:38 +0300 Subject: [PATCH 14/33] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0ded3035b55..39db81c1d2e 100644 --- a/README.md +++ b/README.md @@ -18,4 +18,5 @@ ClickHouse is an open-source column-oriented database management system that all * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date. * [Talks on Saint HighLoad++ in St. Petersburg](https://www.highload.ru/spb/2020/abstracts/6647) on TBD date. * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date. +* [ClickHouse Online Meetup (in Russian)](https://events.yandex.ru/events/click-house-onlajn-vs-03-04-2020) on April 3, 2020. * [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. From cffa45608f33ee1c1bdd8f9fa8f444dfcf0889f1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 12:37:59 +0300 Subject: [PATCH 15/33] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 39db81c1d2e..1f9a5eae75a 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events +* [ClickHouse Online Meetup (in Russian)](https://events.yandex.ru/events/click-house-onlajn-vs-03-04-2020) on April 3, 2020. +* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date. * [Talks on Saint HighLoad++ in St. Petersburg](https://www.highload.ru/spb/2020/abstracts/6647) on TBD date. * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date. -* [ClickHouse Online Meetup (in Russian)](https://events.yandex.ru/events/click-house-onlajn-vs-03-04-2020) on April 3, 2020. -* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. From 3fbc62d23ec097b9ebd9ed58fce8d4af05868b25 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 13:16:18 +0300 Subject: [PATCH 16/33] [docs] try more aggressive httpd shutdown --- docs/tools/build.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 42ea30220d0..04ee2638220 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -258,8 +258,15 @@ def build_single_page_version(lang, args, cfg): logging.info(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) finally: - logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}") - process.terminate() + logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}")) + process.kill() + while True: + time.sleep(0.25) + try: + process.close() + break + except ValueError: + logging.info(f"Waiting for {lang} httpd at port {port_for_pdf} to stop") if not args.version_prefix: # maybe enable in future From 6a48af80218cb58800bd2154788dce4287d35e70 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 13:35:56 +0300 Subject: [PATCH 17/33] Update website.py --- docs/tools/website.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tools/website.py b/docs/tools/website.py index df2db26dd8a..a994982ce96 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -125,8 +125,7 @@ def minify_website(args): '--compilation_level', 'SIMPLE', '--dependency_mode', 'NONE', '--third_party', '--use_types_for_optimization', - '--isolation_mode', 'IIFE', - '--create_source_map', '%outname%.map' + '--isolation_mode', 'IIFE' ] logging.info(closure_args) if closure.run(*closure_args): From cd1683a8238c526184f848ba802260095a1fd884 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 31 Mar 2020 13:38:24 +0300 Subject: [PATCH 18/33] Rename joined table totals columns (#9939) --- dbms/src/Interpreters/SubqueryForSet.cpp | 6 ++- dbms/src/Interpreters/SubqueryForSet.h | 1 + .../Transforms/ExpressionTransform.cpp | 2 +- .../InflatingExpressionTransform.cpp | 2 +- .../01107_join_right_table_totals.reference | 10 +++++ .../01107_join_right_table_totals.sql | 38 +++++++++++++++++++ 6 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference create mode 100644 dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql diff --git a/dbms/src/Interpreters/SubqueryForSet.cpp b/dbms/src/Interpreters/SubqueryForSet.cpp index 47de516d154..98f670e5c64 100644 --- a/dbms/src/Interpreters/SubqueryForSet.cpp +++ b/dbms/src/Interpreters/SubqueryForSet.cpp @@ -53,7 +53,11 @@ bool SubqueryForSet::insertJoinedBlock(Block & block) void SubqueryForSet::setTotals() { if (join && source) - join->setTotals(source->getTotals()); + { + Block totals = source->getTotals(); + renameColumns(totals); + join->setTotals(totals); + } } } diff --git a/dbms/src/Interpreters/SubqueryForSet.h b/dbms/src/Interpreters/SubqueryForSet.h index 3463f708a46..4c99e34a2fc 100644 --- a/dbms/src/Interpreters/SubqueryForSet.h +++ b/dbms/src/Interpreters/SubqueryForSet.h @@ -42,6 +42,7 @@ struct SubqueryForSet private: NamesWithAliases joined_block_aliases; /// Rename column from joined block from this list. + /// Rename source right table column names into qualified column names if they conflicts with left table ones. void renameColumns(Block & block); }; diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.cpp b/dbms/src/Processors/Transforms/ExpressionTransform.cpp index cbc83692c4b..bf52a13f08a 100644 --- a/dbms/src/Processors/Transforms/ExpressionTransform.cpp +++ b/dbms/src/Processors/Transforms/ExpressionTransform.cpp @@ -26,7 +26,7 @@ void ExpressionTransform::transform(Chunk & chunk) { initialized = true; - if (expression->resultIsAlwaysEmpty()) + if (expression->resultIsAlwaysEmpty() && !on_totals) { stopReading(); chunk.clear(); diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp index b5d288197b1..017cfc7cf82 100644 --- a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp @@ -25,7 +25,7 @@ void InflatingExpressionTransform::transform(Chunk & chunk) { initialized = true; - if (expression->resultIsAlwaysEmpty()) + if (expression->resultIsAlwaysEmpty() && !on_totals) { stopReading(); chunk.clear(); diff --git a/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference new file mode 100644 index 00000000000..77db8015b0e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference @@ -0,0 +1,10 @@ + +0 + +0 + +0 + +0 + +0 0 0 diff --git a/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql new file mode 100644 index 00000000000..77e8848c957 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (item_id UInt64, price_sold Float32, date Date) ENGINE MergeTree ORDER BY item_id; + +SELECT item_id +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +FULL JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) r +USING (item_id); + +SELECT id +FROM (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) l +FULL JOIN (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) r +USING (id); + +SELECT item_id +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +INNER JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) r +USING (item_id); + +SELECT id +FROM (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) l +INNER JOIN (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) r +USING (id); + +SELECT id, yago, recent +FROM ( + SELECT item_id AS id, SUM(price_sold) AS recent + FROM t WHERE (date BETWEEN '2019-12-16' AND '2020-03-08') + GROUP BY id WITH TOTALS +) ll +FULL JOIN +( + SELECT item_id AS id, SUM(price_sold) AS yago + FROM t WHERE (date BETWEEN '2018-12-17' AND '2019-03-10') + GROUP BY id WITH TOTALS +) rr +USING (id); + +DROP TABLE t; From c77fe290d36ff0226a18ab37c5fa60b623cae438 Mon Sep 17 00:00:00 2001 From: achulkov2 Date: Tue, 31 Mar 2020 13:52:26 +0300 Subject: [PATCH 19/33] Fix mysql + join bug (#9949) --- dbms/src/Storages/StorageMySQL.cpp | 2 +- dbms/src/Storages/StorageXDBC.cpp | 2 +- ..._transform_query_for_external_database.cpp | 5 ++++- .../transformQueryForExternalDatabase.cpp | 7 +++---- .../transformQueryForExternalDatabase.h | 3 ++- .../test_mysql_database_engine/test.py | 21 +++++++++++++++++++ 6 files changed, 32 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index e0f8cc5813a..6645b41376a 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -71,7 +71,7 @@ Pipes StorageMySQL::read( { check(column_names_); String query = transformQueryForExternalDatabase( - *query_info_.query, getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, context_); + query_info_, getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, context_); Block sample_block; for (const String & column_name : column_names_) diff --git a/dbms/src/Storages/StorageXDBC.cpp b/dbms/src/Storages/StorageXDBC.cpp index 932d7799368..cedd2abf38f 100644 --- a/dbms/src/Storages/StorageXDBC.cpp +++ b/dbms/src/Storages/StorageXDBC.cpp @@ -74,7 +74,7 @@ std::function StorageXDBC::getReadPOSTDataCallback(const N QueryProcessingStage::Enum & /*processed_stage*/, size_t /*max_block_size*/) const { - String query = transformQueryForExternalDatabase(*query_info.query, + String query = transformQueryForExternalDatabase(query_info, getColumns().getOrdinary(), bridge_helper->getIdentifierQuotingStyle(), remote_database_name, diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index abf0b7a6361..385e47201a5 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -49,7 +49,10 @@ static void check(const std::string & query, const std::string & expected, const { ParserSelectQuery parser; ASTPtr ast = parseQuery(parser, query, 1000); - std::string transformed_query = transformQueryForExternalDatabase(*ast, columns, IdentifierQuotingStyle::DoubleQuotes, "test", "table", context); + SelectQueryInfo query_info; + query_info.syntax_analyzer_result = SyntaxAnalyzer(context).analyzeSelect(ast, columns); + query_info.query = ast; + std::string transformed_query = transformQueryForExternalDatabase(query_info, columns, IdentifierQuotingStyle::DoubleQuotes, "test", "table", context); EXPECT_EQ(transformed_query, expected); } diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index 5ba7deadb33..839175c9b72 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -134,16 +134,15 @@ bool isCompatible(const IAST & node) String transformQueryForExternalDatabase( - const IAST & query, + const SelectQueryInfo & query_info, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, const String & table, const Context & context) { - auto clone_query = query.clone(); - auto syntax_result = SyntaxAnalyzer(context).analyzeSelect(clone_query, available_columns); - const Names used_columns = syntax_result->requiredSourceColumns(); + auto clone_query = query_info.query->clone(); + const Names used_columns = query_info.syntax_analyzer_result->requiredSourceColumns(); auto select = std::make_shared(); diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.h b/dbms/src/Storages/transformQueryForExternalDatabase.h index 9b838da5127..09de53e36d3 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.h +++ b/dbms/src/Storages/transformQueryForExternalDatabase.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -23,7 +24,7 @@ class Context; * Compatible expressions are comparisons of identifiers, constants, and logical operations on them. */ String transformQueryForExternalDatabase( - const IAST & query, + const SelectQueryInfo & query_info, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, diff --git a/dbms/tests/integration/test_mysql_database_engine/test.py b/dbms/tests/integration/test_mysql_database_engine/test.py index 9e776b65f32..86e0b9df5fd 100644 --- a/dbms/tests/integration/test_mysql_database_engine/test.py +++ b/dbms/tests/integration/test_mysql_database_engine/test.py @@ -99,3 +99,24 @@ def test_clickhouse_dml_for_mysql_database(started_cluster): assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '10000' mysql_node.query("DROP DATABASE test_database") + + +def test_clickhouse_join_for_mysql_database(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + mysql_node.query("CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'") + mysql_node.query("CREATE TABLE test.t1_mysql_local (" + "pays VARCHAR(55) DEFAULT 'FRA' NOT NULL," + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco CHAR(3) DEFAULT '' NOT NULL" + ")") + mysql_node.query("CREATE TABLE test.t2_mysql_local (" + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco VARCHAR(5) DEFAULT ''" + ")") + clickhouse_node.query("CREATE TABLE default.t1_remote_mysql AS mysql('mysql1:3306','test','t1_mysql_local','root','clickhouse')") + clickhouse_node.query("CREATE TABLE default.t2_remote_mysql AS mysql('mysql1:3306','test','t2_mysql_local','root','clickhouse')") + assert clickhouse_node.query("SELECT s.pays " + "FROM default.t1_remote_mysql AS s " + "LEFT JOIN default.t1_remote_mysql AS s_ref " + "ON (s_ref.opco = s.opco AND s_ref.service = s.service)") == '' + mysql_node.query("DROP DATABASE test") From 5b414ff5096bc24165f396210a0fb00132e0f55e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 13:56:38 +0300 Subject: [PATCH 20/33] fix mistype --- docs/tools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 04ee2638220..4ba9a82d7b2 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -258,7 +258,7 @@ def build_single_page_version(lang, args, cfg): logging.info(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) finally: - logging.info(f"Stop serving for {lang} pdf at port {port_for_pdf}")) + logging.info(f'Stop serving for {lang} pdf at port {port_for_pdf}') process.kill() while True: time.sleep(0.25) @@ -266,7 +266,7 @@ def build_single_page_version(lang, args, cfg): process.close() break except ValueError: - logging.info(f"Waiting for {lang} httpd at port {port_for_pdf} to stop") + logging.info(f'Waiting for {lang} httpd at port {port_for_pdf} to stop') if not args.version_prefix: # maybe enable in future From 06a375f49e0ada42ed87dd496900fc9f9077362c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 14:17:28 +0300 Subject: [PATCH 21/33] Update a Slack link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f9a5eae75a..aab4cb9f63c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ClickHouse is an open-source column-oriented database management system that all * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster. * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. +* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. From 0dae1c27ec36bb4308b276eb1e36a01b3f310673 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 14:18:21 +0300 Subject: [PATCH 22/33] Update a Slack link --- docs/tools/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/README.md b/docs/tools/README.md index 56ca016ad9e..11e04de4b6a 100644 --- a/docs/tools/README.md +++ b/docs/tools/README.md @@ -38,5 +38,5 @@ The easiest way to see the result is to use `--livereload=8888` argument of buil At the moment there’s no easy way to do just that, but you can consider: -- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk). +- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng). - Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for https://clickhouse.tech. From 460cfc96e2d71d89a06b603bc95f8ea8e1bde6eb Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 14:22:56 +0300 Subject: [PATCH 23/33] Update mdx_clickhouse.py --- docs/tools/mdx_clickhouse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index f024c129462..50fe6ca4f15 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -11,6 +11,7 @@ import macros.plugin import slugify as slugify_impl + class ClickHouseLinkMixin(object): def handleMatch(self, m, data): From 487c088ab7a420da42ac2db9d1b5587b640ae5fc Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 15:20:03 +0300 Subject: [PATCH 24/33] Temporary make wkhtmltopdf more verbose --- docs/tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 4ba9a82d7b2..2a3111a7fda 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -249,7 +249,7 @@ def build_single_page_version(lang, args, cfg): create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', - '--log-level', 'warn', + # TODO: restore '--log-level', 'w', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] try: From a2ceb929074430e513f2637ee90d08d55ea1b56e Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 16:04:53 +0300 Subject: [PATCH 25/33] Update concatenate.py --- docs/tools/concatenate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/tools/concatenate.py b/docs/tools/concatenate.py index cfbc2acbb4c..65efe9bb5eb 100755 --- a/docs/tools/concatenate.py +++ b/docs/tools/concatenate.py @@ -14,9 +14,8 @@ def concatenate(lang, docs_path, single_page_file): files_to_concatenate = [(l[l.index(':') + 1:]).strip(" '\n") for l in cfg_file if '.md' in l and 'single_page' not in l] - logging.info( - str(len(files_to_concatenate)) + - ' files will be concatenated into single md-file.') + files_count = len(files_to_concatenate) + logging.info(f'{files_count} files will be concatenated into single md-file for {lang}.') logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) for path in files_to_concatenate: From 54277793c2d0d6bc0ba4b21231800012dff11ef7 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 16:31:16 +0300 Subject: [PATCH 26/33] [docs] unprallelize temporarily --- docs/tools/util.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/tools/util.py b/docs/tools/util.py index 3dc58807612..c058442fa29 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -44,16 +44,19 @@ def get_free_port(): def run_function_in_parallel(func, args_list, threads=False): - processes = [] - exit_code = 0 for task in args_list: - cls = threading.Thread if threads else multiprocessing.Process - processes.append(cls(target=func, args=task)) - processes[-1].start() - for process in processes: - process.join() - if not threads: - if process.exitcode and not exit_code: - exit_code = process.exitcode - if exit_code: - sys.exit(exit_code) + func(*args_list) +# TODO: back to parallel +# processes = [] +# exit_code = 0 +# for task in args_list: +# cls = threading.Thread if threads else multiprocessing.Process +# processes.append(cls(target=func, args=task)) +# processes[-1].start() +# for process in processes: +# process.join() +# if not threads: +# if process.exitcode and not exit_code: +# exit_code = process.exitcode +# if exit_code: +# sys.exit(exit_code) From b5a36dd1b147b4e00fb2ee5e8bd7a95f83453080 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 16:34:57 +0300 Subject: [PATCH 27/33] Update util.py --- docs/tools/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/util.py b/docs/tools/util.py index c058442fa29..a746a50db1d 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -45,7 +45,7 @@ def get_free_port(): def run_function_in_parallel(func, args_list, threads=False): for task in args_list: - func(*args_list) + func(*task) # TODO: back to parallel # processes = [] # exit_code = 0 From ad1ab0a8f2110d05881c80786cead907114a7d8a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 17:21:24 +0300 Subject: [PATCH 28/33] Adjust wkhtmltopdf args --- docs/tools/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 2a3111a7fda..56cb19f56fc 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -249,7 +249,8 @@ def build_single_page_version(lang, args, cfg): create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', - # TODO: restore '--log-level', 'w', + '--use-xserver', '--javascript-delay', '3000', + # TODO: restore '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] try: From 9477864cd6f887a906c71ccc11a3c7302f3c94dd Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 18:37:48 +0300 Subject: [PATCH 29/33] Update build.py --- docs/tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 56cb19f56fc..28d749d48cb 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -249,7 +249,7 @@ def build_single_page_version(lang, args, cfg): create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', - '--use-xserver', '--javascript-delay', '3000', + '--javascript-delay', '3000', # TODO: restore '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] From 2d640a9bf821f9feb08c3fe5dbfde206b11a0c8b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 18:56:49 +0300 Subject: [PATCH 30/33] Update build.py --- docs/tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 28d749d48cb..e841c6a212e 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -249,7 +249,7 @@ def build_single_page_version(lang, args, cfg): create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', - '--javascript-delay', '3000', + '--disable-javascript', # TODO: restore '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] From a7f8b8acde8b2694d2629ee79054123b92c28be1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 31 Mar 2020 20:07:21 +0300 Subject: [PATCH 31/33] Workaround for top nav issue --- website/css/bootstrap.css | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/website/css/bootstrap.css b/website/css/bootstrap.css index 8eac957a512..dcfed4306e6 100644 --- a/website/css/bootstrap.css +++ b/website/css/bootstrap.css @@ -7978,15 +7978,14 @@ button.bg-dark:focus { z-index: 1030; } -@supports ((position: -webkit-sticky) or (position: sticky)) { - .sticky-top { - position: -webkit-sticky; - position: sticky; - top: 0; - z-index: 1020; - } +.sticky-top { + position: -webkit-sticky; + position: sticky; + top: 0; + z-index: 1020; } + .sr-only { position: absolute; width: 1px; @@ -10221,4 +10220,4 @@ a.text-dark:hover, a.text-dark:focus { border-color: #dee2e6; } } -/*# sourceMappingURL=bootstrap.css.map */ \ No newline at end of file +/*# sourceMappingURL=bootstrap.css.map */ From c424db85b463a72eda311fa4848a6876440a9bd8 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Tue, 31 Mar 2020 20:25:13 +0300 Subject: [PATCH 32/33] fix timeout in 01098_temporary_and_external_tables --- dbms/src/Common/QueryProfiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/QueryProfiler.cpp b/dbms/src/Common/QueryProfiler.cpp index 4bf5009d51c..ac2987a3795 100644 --- a/dbms/src/Common/QueryProfiler.cpp +++ b/dbms/src/Common/QueryProfiler.cpp @@ -40,7 +40,7 @@ namespace if (overrun_count) { /// But pass with some frequency to avoid drop of all traces. - if (write_trace_iteration % overrun_count == 0) + if (write_trace_iteration % (overrun_count + 1) == 0) { ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, overrun_count); } From 2b235f1b1abf70bda2a563ea2c7977c4c9fa1188 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Mar 2020 22:33:37 +0300 Subject: [PATCH 33/33] Remove dynamic cast from check --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 90c0d3418e6..336fef069d0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3336,7 +3336,7 @@ void StorageReplicatedMergeTree::alter( } else if (rc == Coordination::ZBADVERSION) { - if (dynamic_cast(*results[0]).error) + if (results[0]->error) throw Exception("Metadata on replica is not up to date with common metadata in Zookeeper. Cannot alter", ErrorCodes::CANNOT_ASSIGN_ALTER); continue;