From dd3ea8d31b1b5615bcdd6b671f5e6e86315e8dda Mon Sep 17 00:00:00 2001 From: liyang830 Date: Fri, 10 Mar 2023 20:52:27 +0800 Subject: [PATCH 001/332] feat: modify materalized view query, check inner table structure --- src/Storages/StorageMaterializedView.cpp | 13 +++++++++++++ ...erialized_view_query_has_inner_table.reference | 2 ++ ...er_materialized_view_query_has_inner_table.sql | 15 +++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference create mode 100644 tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index ae3fa62b38c..0bbd689043c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -269,6 +270,18 @@ void StorageMaterializedView::alter( DatabaseCatalog::instance().updateViewDependency(old_select.select_table_id, table_id, new_select.select_table_id, table_id); new_metadata.setSelectQuery(new_select); + + /// check materialized view inner table structure + if (has_inner_table) + { + const Block & block = InterpreterSelectWithUnionQuery::getSampleBlock(new_select.select_query, local_context); + for (const auto & col : block.getColumnsWithTypeAndName()) + { + if (!tryGetTargetTable()->getInMemoryMetadata().columns.has(col.name)) + throw Exception(ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW, "column {} is not in materialized view inner table", col.name); + } + } + } /// end modify query diff --git a/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference new file mode 100644 index 00000000000..1191247b6d9 --- /dev/null +++ b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.reference @@ -0,0 +1,2 @@ +1 +2 diff --git a/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql new file mode 100644 index 00000000000..73bbac59a95 --- /dev/null +++ b/tests/queries/0_stateless/25340_alter_materialized_view_query_has_inner_table.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS src_table; +DROP TABLE IF EXISTS mv; + +CREATE TABLE src_table (`a` UInt32, `b` UInt32) ENGINE = MergeTree ORDER BY a; +CREATE MATERIALIZED VIEW mv UUID '2bad6d75-86fe-4da0-815b-2c7410253941' (`a` UInt32) ENGINE = MergeTree ORDER BY a AS SELECT a FROM src_table; + +INSERT INTO src_table (a, b) VALUES (1, 1), (2, 2); + +SELECT * FROM mv; + +SET allow_experimental_alter_materialized_view_structure = 1; +ALTER TABLE mv MODIFY QUERY SELECT a, b FROM src_table; -- {serverError QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW} + +DROP TABLE src_table; +DROP TABLE mv; \ No newline at end of file From 3f5853b970dd205465a5593d5786c1f8a4d82cc7 Mon Sep 17 00:00:00 2001 From: AN Date: Fri, 27 Oct 2023 19:17:13 +0300 Subject: [PATCH 002/332] Update index.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Punctuation fixes, узел→сервер as suggested by alexei-milovidov at https://github.com/ClickHouse/ClickHouse/pull/56040#issuecomment-1783155867, консистентность → согласованность (standard translation instead of calque) --- docs/ru/index.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/ru/index.md b/docs/ru/index.md index 78bb382753b..a9a666b18db 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -41,7 +41,7 @@ ClickHouse — столбцовая система управления база Разный порядок хранения данных лучше подходит для разных сценариев работы. Сценарий работы с данными — это то, какие производятся запросы, как часто и в каком соотношении; сколько читается данных на запросы каждого вида — строк, столбцов, байтов; как соотносятся чтения и обновления данных; какой рабочий размер данных и насколько локально он используется; используются ли транзакции и с какой изолированностью; какие требования к дублированию данных и логической целостности; требования к задержкам на выполнение и пропускной способности запросов каждого вида и т. п. -Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке, система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. +Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы. ## Ключевые особенности OLAP-сценария работы {#kliuchevye-osobennosti-olap-stsenariia-raboty} @@ -53,11 +53,11 @@ ClickHouse — столбцовая система управления база - запросы идут сравнительно редко (обычно не более сотни в секунду на сервер); - при выполнении простых запросов, допустимы задержки в районе 50 мс; - значения в столбцах достаточно мелкие — числа и небольшие строки (например, 60 байт на URL); -- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один узел); +- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один сервер); - транзакции отсутствуют; -- низкие требования к консистентности данных; -- в запросе одна большая таблица, все таблицы кроме одной маленькие; -- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного узла. +- низкие требования к согласованности данных; +- в запросе одна большая таблица, все остальные таблицы из запроса — маленькие; +- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного сервера. Легко видеть, что OLAP-сценарий работы существенно отличается от других распространённых сценариев работы (например, OLTP или Key-Value сценариев работы). Таким образом, не имеет никакого смысла пытаться использовать OLTP-системы или системы класса «ключ — значение» для обработки аналитических запросов, если вы хотите получить приличную производительность («выше плинтуса»). Например, если вы попытаетесь использовать для аналитики MongoDB или Redis — вы получите анекдотически низкую производительность по сравнению с OLAP-СУБД. @@ -77,11 +77,11 @@ ClickHouse — столбцовая система управления база ### По вводу-выводу {#po-vvodu-vyvodu} -1. Для выполнения аналитического запроса, требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. -2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. -3. За счёт уменьшения ввода-вывода, больше данных влезает в системный кэш. +1. Для выполнения аналитического запроса требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода. +2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам, также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода. +3. За счёт уменьшения ввода-вывода больше данных влезает в системный кэш. -Например, для запроса «посчитать количество записей для каждой рекламной системы», требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается. +Например, для запроса «посчитать количество записей для каждой рекламной системы» требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике такая скорость действительно достигается. ### По вычислениям {#po-vychisleniiam} From 4dcbd6775a9cd1afe3c8be96e3c68c397ae547f0 Mon Sep 17 00:00:00 2001 From: Thom O'Connor Date: Fri, 3 Nov 2023 09:46:35 -0600 Subject: [PATCH 003/332] Update kill.md Added additional examples and context for killing queries and mutations --- docs/en/sql-reference/statements/kill.md | 64 ++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 294724dfa50..32de7a41e72 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -21,6 +21,35 @@ The queries to terminate are selected from the system.processes table using the Examples: +First, you'll need to get the list of incomplete queries. This SQL queries provides them according to those running the longest: + +List from a single ClickHouse node: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM system.processes + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +List from a ClickHouse cluster: +``` sql +SELECT + initial_query_id, + query_id, + formatReadableTimeDelta(elapsed) AS time_delta, + query, + * + FROM clusterAllReplicas(default, system.processes) + WHERE query ILIKE 'SELECT%' + ORDER BY time_delta DESC; +``` + +Kill the query: ``` sql -- Forcibly terminates all queries with the specified query_id: KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' @@ -44,6 +73,8 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que ## KILL MUTATION +One of the first things to check if a ClickHouse system or service is not running well is for long-running, incomplete mutations. The asynchronous (background) nature of mutations can cause a large queue of them that can then consume all available resources on the service. You may need to either pause all new mutations, INSERTs, and SELECTs and allow the queue of mutations to complete, or else manually kill some of these mutations. + ``` sql KILL MUTATION [ON CLUSTER cluster] WHERE @@ -57,6 +88,39 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut Examples: +Get a count() of the number of incomplete mutations: + +Count of mutations from a single ClickHouse node: +``` sql +SELECT count(*) +FROM system.mutations +WHERE is_done = 0; +``` + +Count of mutations from a ClickHouse cluster of replicas: +``` sql +SELECT count(*) +FROM clusterAllReplicas('default',system.mutations) +WHERE is_done = 0; +``` + +Query the list of incomplete mutations: + +List of mutations from a single ClickHouse node: +``` sql +SELECT mutation_id,* +FROM system.mutations +WHERE is_done = 0; +``` + +List of mutations from a ClickHouse cluster: +``` sql +SELECT mutation_id,* +FROM clusterAllReplicas('default',system.mutations) +WHERE is_done = 0; +``` + +Kill the mutations as needed: ``` sql -- Cancel and remove all mutations of the single table: KILL MUTATION WHERE database = 'default' AND table = 'table' From 1134af19caeaffcf70cc94146faed346d6af0cf6 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:33:29 -0800 Subject: [PATCH 004/332] [Docs] Fix typo Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 32de7a41e72..a7d050e548c 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -21,7 +21,7 @@ The queries to terminate are selected from the system.processes table using the Examples: -First, you'll need to get the list of incomplete queries. This SQL queries provides them according to those running the longest: +First, you'll need to get the list of incomplete queries. This SQL query provides them according to those running the longest: List from a single ClickHouse node: ``` sql From 7e0d95e48cb399c047c9756d81b0f76ce67ea57f Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:33:43 -0800 Subject: [PATCH 005/332] [Docs] Formatting Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index a7d050e548c..57448c4f441 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -88,7 +88,7 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut Examples: -Get a count() of the number of incomplete mutations: +Get a `count()` of the number of incomplete mutations: Count of mutations from a single ClickHouse node: ``` sql From ec02a2a2c4f4d4a279732df2c2dd61ab8b0cb80a Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Tue, 7 Nov 2023 22:36:49 -0800 Subject: [PATCH 006/332] [Docs] Reword for clarity Co-authored-by: Johnny <9611008+johnnymatthews@users.noreply.github.com> --- docs/en/sql-reference/statements/kill.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 57448c4f441..c85870fc0c8 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -73,7 +73,10 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que ## KILL MUTATION -One of the first things to check if a ClickHouse system or service is not running well is for long-running, incomplete mutations. The asynchronous (background) nature of mutations can cause a large queue of them that can then consume all available resources on the service. You may need to either pause all new mutations, INSERTs, and SELECTs and allow the queue of mutations to complete, or else manually kill some of these mutations. +The presence of long-running or incomplete mutations often indicates that a ClickHouse service is running poorly. The asynchronous nature of mutations can cause them to consume all available resources on a system. You may need to either: + +- Pause all new mutations, `INSERT`s , and `SELECT`s and allow the queue of mutations to complete. +- Or manually kill some of these mutations by sending a `KILLSIG` command. ``` sql KILL MUTATION [ON CLUSTER cluster] From 039bb1d599a5262e558b9b4ebd66fd85469afa3c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 22 Jan 2024 20:26:28 +0100 Subject: [PATCH 007/332] fix race on Context::async_insert_queue --- src/Interpreters/Context.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7e89c794712..51cfd302338 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -181,6 +181,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; extern const int CLUSTER_DOESNT_EXIST; + extern const int ABORTED; } #define SHUTDOWN(log, desc, ptr, method) do \ @@ -556,7 +557,12 @@ struct ContextSharedPart : boost::noncopyable return; /// Need to flush the async insert queue before shutting down the database catalog - async_insert_queue.reset(); + std::shared_ptr delete_async_insert_queue; + { + std::lock_guard lock(mutex); + delete_async_insert_queue = std::move(async_insert_queue); + } + delete_async_insert_queue.reset(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config @@ -4838,11 +4844,15 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const { - return shared->async_insert_queue.get(); + std::lock_guard lock(mutex); + if (auto res = shared->async_insert_queue.get()) + return res; + throw Exception(ErrorCodes::ABORTED, "AsynchronousInsertQueue is not initialized yet or has been already shutdown"); } void Context::setAsynchronousInsertQueue(const std::shared_ptr & ptr) { + std::lock_guard lock(mutex); using namespace std::chrono; if (std::chrono::milliseconds(settings.async_insert_busy_timeout_ms) == 0ms) From eb881667638524f182f06e19d699704ce9e86196 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jan 2024 00:28:28 +0100 Subject: [PATCH 008/332] Update Context.cpp --- src/Interpreters/Context.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 51cfd302338..217b247c21c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -181,7 +181,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_COLUMNS_DOESNT_MATCH; extern const int CLUSTER_DOESNT_EXIST; - extern const int ABORTED; } #define SHUTDOWN(log, desc, ptr, method) do \ @@ -4845,9 +4844,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const { std::lock_guard lock(mutex); - if (auto res = shared->async_insert_queue.get()) - return res; - throw Exception(ErrorCodes::ABORTED, "AsynchronousInsertQueue is not initialized yet or has been already shutdown"); + return shared->async_insert_queue.get(); } void Context::setAsynchronousInsertQueue(const std::shared_ptr & ptr) From 4cfc8d1a34342d44adbc7d9c8c3a4916670d68b2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Jan 2024 00:30:42 +0100 Subject: [PATCH 009/332] better method name --- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- src/Interpreters/executeQuery.cpp | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/System/StorageSystemAsynchronousInserts.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 217b247c21c..ab42e6b0ec9 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4841,7 +4841,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const return ignored_part_uuids; } -AsynchronousInsertQueue * Context::getAsynchronousInsertQueue() const +AsynchronousInsertQueue * Context::tryGetAsynchronousInsertQueue() const { std::lock_guard lock(mutex); return shared->async_insert_queue.get(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 65566876a80..6180bfbde88 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1203,7 +1203,7 @@ public: PartUUIDsPtr getPartUUIDs() const; PartUUIDsPtr getIgnoredPartUUIDs() const; - AsynchronousInsertQueue * getAsynchronousInsertQueue() const; + AsynchronousInsertQueue * tryGetAsynchronousInsertQueue() const; void setAsynchronousInsertQueue(const std::shared_ptr & ptr); ReadTaskCallback getReadTaskCallback() const; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 1712c9608bf..f478b43049f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -692,7 +692,7 @@ BlockIO InterpreterSystemQuery::execute() case Type::FLUSH_ASYNC_INSERT_QUEUE: { getContext()->checkAccess(AccessType::SYSTEM_FLUSH_ASYNC_INSERT_QUEUE); - auto * queue = getContext()->getAsynchronousInsertQueue(); + auto * queue = getContext()->tryGetAsynchronousInsertQueue(); if (!queue) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot flush asynchronous insert queue because it is not initialized"); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 4b5a6a84e17..a84c957d9a8 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -924,7 +924,7 @@ static std::tuple executeQueryImpl( std::unique_ptr interpreter; bool async_insert = false; - auto * queue = context->getAsynchronousInsertQueue(); + auto * queue = context->tryGetAsynchronousInsertQueue(); auto * logger = &Poco::Logger::get("executeQuery"); if (insert_query && async_insert_enabled) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index fa7206eeaac..9bc6c3872fd 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -907,7 +907,7 @@ void TCPHandler::processInsertQuery() Block processed_block; const auto & settings = query_context->getSettingsRef(); - auto * insert_queue = query_context->getAsynchronousInsertQueue(); + auto * insert_queue = query_context->tryGetAsynchronousInsertQueue(); const auto & insert_query = assert_cast(*state.parsed_query); bool async_insert_enabled = settings.async_insert; diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.cpp b/src/Storages/System/StorageSystemAsynchronousInserts.cpp index 20ba4d1cdfb..b480821f8ea 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.cpp +++ b/src/Storages/System/StorageSystemAsynchronousInserts.cpp @@ -34,7 +34,7 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co { using namespace std::chrono; - auto * insert_queue = context->getAsynchronousInsertQueue(); + auto * insert_queue = context->tryGetAsynchronousInsertQueue(); if (!insert_queue) return; From f91feb0dcb405df80f317f456372c7374f2c75ee Mon Sep 17 00:00:00 2001 From: Daniil Ivanik Date: Tue, 30 Jan 2024 14:17:11 +0100 Subject: [PATCH 010/332] Initial working commit --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 210 ++++++++++-------- src/Storages/SelectQueryInfo.h | 2 +- src/Storages/System/StorageSystemNumbers.cpp | 8 +- src/Storages/System/StorageSystemNumbers.h | 36 +-- src/Storages/System/attachSystemTables.cpp | 5 +- src/TableFunctions/CMakeLists.txt | 2 +- src/TableFunctions/ITableFunction.cpp | 2 +- src/TableFunctions/TableFunctionNumbers.cpp | 2 +- .../TableFunctionsGenerateSeries.cpp | 100 +++++++++ src/TableFunctions/registerTableFunctions.cpp | 1 + src/TableFunctions/registerTableFunctions.h | 1 + 11 files changed, 234 insertions(+), 135 deletions(-) create mode 100644 src/TableFunctions/TableFunctionsGenerateSeries.cpp diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 5173b18c6bf..13a14ffb917 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -26,41 +26,59 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_) - : ISource(createHeader()), block_size(block_size_), next(offset_), step(step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string& column_name, UInt64 inner_step_) + : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) { } String getName() const override { return "Numbers"; } - static Block createHeader() { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number")}; } + static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } protected: Chunk generate() override { - auto column = ColumnUInt64::create(block_size); - ColumnUInt64::Container & vec = column->getData(); UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. + UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; + if (first_element < curr) { + first_element += inner_step; + } + UInt64 filtered_block_size = 0; + if (first_element - curr >= block_size) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, filtered_block_size}; + } + if (first_element - curr < block_size) { + filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; + } + + auto column = ColumnUInt64::create(filtered_block_size); + ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. - UInt64 * end = &vec[block_size]; - iota(pos, static_cast(end - pos), curr); + UInt64 * end = &vec[filtered_block_size]; + iota(pos, static_cast(end - pos), UInt64{0}); + for (UInt64 p = 0; p < filtered_block_size; p += 1) { + vec[p] = vec[p] * inner_step + first_element; + } next += step; progress(column->size(), column->byteSize()); - return {Columns{std::move(column)}, block_size}; + return {Columns{std::move(column)}, filtered_block_size}; } private: UInt64 block_size; UInt64 next; UInt64 step; + UInt64 inner_step; + UInt64 inner_remainder; }; -UInt128 sizeOfRange(const Range & r) +[[maybe_unused]] UInt128 sizeOfRange(const Range & r) { UInt128 size; if (r.right.isPositiveInfinity()) @@ -77,7 +95,7 @@ UInt128 sizeOfRange(const Range & r) return size; }; -auto sizeOfRanges(const Ranges & rs) +[[maybe_unused]] auto sizeOfRanges(const Ranges & rs) { UInt128 total_size{}; for (const Range & r : rs) @@ -91,7 +109,7 @@ auto sizeOfRanges(const Ranges & rs) /// Generate numbers according to ranges. /// Numbers generated is ordered in one stream. /// Notice that we will not generate additional numbers out of ranges. -class NumbersRangedSource : public ISource +class [[maybe_unused]] NumbersRangedSource : public ISource { public: /// Represent a position in Ranges list. @@ -109,8 +127,8 @@ public: using RangesStatePtr = std::shared_ptr; - NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_) - : ISource(NumbersSource::createHeader()), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) + [[maybe_unused]] NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) + : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) { } @@ -273,7 +291,7 @@ private: namespace { /// Whether we should push limit down to scan. -bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) +[[maybe_unused]] bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) { const auto & query = query_info.query->as(); /// Just ignore some minor cases, such as: @@ -286,7 +304,7 @@ bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -void shrinkRanges(Ranges & ranges, size_t size) +[[maybe_unused]] void shrinkRanges(Ranges & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -375,107 +393,107 @@ Pipe ReadFromSystemNumbersStep::makePipe() num_streams = 1; /// Build rpn of query filters - KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); + // KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); Pipe pipe; Ranges ranges; - if (condition.extractPlainRanges(ranges)) - { - /// Intersect ranges with table range - std::optional table_range; - std::optional overflowed_table_range; + // if (condition.extractPlainRanges(ranges)) + // { + // /// Intersect ranges with table range + // std::optional table_range; + // std::optional overflowed_table_range; - if (numbers_storage.limit.has_value()) - { - if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) - { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); - } - /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) - else - { - table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); - auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - overflowed_table_range.emplace( - FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); - } - } - else - { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); - } + // if (numbers_storage.limit.has_value()) + // { + // if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + // } + // /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) + // else + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); + // auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); + // overflowed_table_range.emplace( + // FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); + // } + // } + // else + // { + // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); + // } - Ranges intersected_ranges; - for (auto & r : ranges) - { - auto intersected_range = table_range->intersectWith(r); - if (intersected_range) - intersected_ranges.push_back(*intersected_range); - } - /// intersection with overflowed_table_range goes back. - if (overflowed_table_range.has_value()) - { - for (auto & r : ranges) - { - auto intersected_range = overflowed_table_range->intersectWith(r); - if (intersected_range) - intersected_ranges.push_back(*overflowed_table_range); - } - } + // Ranges intersected_ranges; + // for (auto & r : ranges) + // { + // auto intersected_range = table_range->intersectWith(r); + // if (intersected_range) + // intersected_ranges.push_back(*intersected_range); + // } + // /// intersection with overflowed_table_range goes back. + // if (overflowed_table_range.has_value()) + // { + // for (auto & r : ranges) + // { + // auto intersected_range = overflowed_table_range->intersectWith(r); + // if (intersected_range) + // intersected_ranges.push_back(*overflowed_table_range); + // } + // } - /// ranges is blank, return a source who has no data - if (intersected_ranges.empty()) - { - pipe.addSource(std::make_shared(NumbersSource::createHeader())); - return pipe; - } - const auto & limit_length = limit_length_and_offset.first; - const auto & limit_offset = limit_length_and_offset.second; + // /// ranges is blank, return a source who has no data + // if (intersected_ranges.empty()) + // { + // pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + // return pipe; + // } + // const auto & limit_length = limit_length_and_offset.first; + // const auto & limit_offset = limit_length_and_offset.second; - /// If intersected ranges is limited or we can pushdown limit. - if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) - { - UInt128 total_size = sizeOfRanges(intersected_ranges); - UInt128 query_limit = limit_length + limit_offset; + // /// If intersected ranges is limited or we can pushdown limit. + // if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) + // { + // UInt128 total_size = sizeOfRanges(intersected_ranges); + // UInt128 query_limit = limit_length + limit_offset; - /// limit total_size by query_limit - if (should_pushdown_limit && query_limit < total_size) - { - total_size = query_limit; - /// We should shrink intersected_ranges for case: - /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - shrinkRanges(intersected_ranges, total_size); - } + // /// limit total_size by query_limit + // if (should_pushdown_limit && query_limit < total_size) + // { + // total_size = query_limit; + // /// We should shrink intersected_ranges for case: + // /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + // shrinkRanges(intersected_ranges, total_size); + // } - checkLimits(size_t(total_size)); + // checkLimits(size_t(total_size)); - if (total_size / max_block_size < num_streams) - num_streams = static_cast(total_size / max_block_size); + // if (total_size / max_block_size < num_streams) + // num_streams = static_cast(total_size / max_block_size); - if (num_streams == 0) - num_streams = 1; + // if (num_streams == 0) + // num_streams = 1; - /// Ranges state, all streams will share the state. - auto ranges_state = std::make_shared(); - for (size_t i = 0; i < num_streams; ++i) - { - auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size); + // /// Ranges state, all streams will share the state. + // auto ranges_state = std::make_shared(); + // for (size_t i = 0; i < num_streams; ++i) + // { + // auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); - if (i == 0) - source->addTotalRowsApprox(total_size); + // if (i == 0) + // source->addTotalRowsApprox(total_size); - pipe.addSource(std::move(source)); - } - return pipe; - } - } + // pipe.addSource(std::move(source)); + // } + // return pipe; + // } + // } /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) { auto source - = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size); + = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 662a5c0ef5a..2b4afaa6345 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -229,4 +229,4 @@ struct SelectQueryInfo bool isFinal() const; }; -} + } diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index b100be7cdf4..cd7207917a9 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -11,15 +11,16 @@ #include #include #include +#include namespace DB { -StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, std::optional limit_, UInt64 offset_) - : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_) +StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name_, std::optional limit_, UInt64 offset_, UInt64 step_) + : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(ColumnsDescription({{"number", std::make_shared()}})); + storage_metadata.setColumns(ColumnsDescription({{column_name_, std::make_shared()}})); setInMemoryMetadata(storage_metadata); } @@ -33,6 +34,7 @@ void StorageSystemNumbers::read( size_t max_block_size, size_t num_streams) { + // LOG_DEBUG(&Poco::Logger::get("Reading from SystemNumbers"), "Limit : {}", limit.value()); query_plan.addStep(std::make_unique( column_names, shared_from_this(), storage_snapshot, query_info, std::move(context), max_block_size, num_streams)); } diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index fe6227db406..ffe87b8ad14 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,39 +10,11 @@ namespace DB class Context; - -/** Implements a table engine for the system table "numbers". - * The table contains the only column number UInt64. - * From this table, you can read all natural numbers, starting from 0 (to 2^64 - 1, and then again). - * - * You could also specify a limit (how many numbers to give). - * - * How to generate numbers? - * - * 1. First try a smart fashion: - * - * In this fashion we try to push filters and limit down to scanning. - * Firstly extract plain ranges(no overlapping and ordered) by filter expressions. - * - * For example: - * where (numbers > 1 and numbers < 3) or (numbers in (4, 6)) or (numbers > 7 and numbers < 9) - * - * We will get ranges - * (1, 3), [4, 4], [6, 6], (7, 9) - * - * Then split the ranges evenly to one or multi-streams. With this way we will get result without large scanning. - * - * 2. If fail to extract plain ranges, fall back to ordinary scanning. - * - * If multithreaded is specified, numbers will be generated in several streams - * (and result could be out of order). If both multithreaded and limit are specified, - * the table could give you not exactly 1..limit range, but some arbitrary 'limit' numbers. - */ -class StorageSystemNumbers final : public IStorage +class StorageSystemNumbers final : public IStorage { public: /// Otherwise, streams concurrently increment atomic. - StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, std::optional limit_ = std::nullopt, UInt64 offset_ = 0); + StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name, std::optional limit_ = std::nullopt, UInt64 offset_ = 0, UInt64 step_ = 1); std::string getName() const override { return "SystemNumbers"; } @@ -67,6 +39,10 @@ private: bool multithreaded; std::optional limit; UInt64 offset; + std::string column_name; + + UInt64 step; + }; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index bf898f57833..ddd89709b6a 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -118,8 +118,9 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attach(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false); - attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true); + attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); + attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); + // attach(context, system_database, "generate_series", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", false, "generate_series"); attach(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attach(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 770990cc405..c5c2a660935 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -29,7 +29,7 @@ if (TARGET ch_contrib::azure_sdk) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::azure_sdk) endif () -if (TARGET ch_contrib::simdjson) +if (TARGET ch_co`trib::simdjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::simdjson) endif () diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 137e1dc27fe..c854b6b0f9c 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -5,7 +5,7 @@ #include #include #include - +#include namespace ProfileEvents { diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 262018f108c..71a9ba097c6 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -63,7 +63,7 @@ StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_f UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, length, offset); + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionsGenerateSeries.cpp b/src/TableFunctions/TableFunctionsGenerateSeries.cpp new file mode 100644 index 00000000000..3941f1eadb2 --- /dev/null +++ b/src/TableFunctions/TableFunctionsGenerateSeries.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "registerTableFunctions.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +/* numbers(limit), numbers_mt(limit) + * - the same as SELECT number FROM system.numbers LIMIT limit. + * Used for testing purposes, as a simple example of table function. + */ +class TableFunctionGenerateSeries : public ITableFunction +{ +public: + static constexpr auto name = "generate_series"; + std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } +private: + StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + const char * getStorageTypeName() const override { return "SystemNumbers"; } + + UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; + + ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; +}; + +ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const +{ + /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 + return ColumnsDescription{{{"generate_series", std::make_shared()}}}; +} + +StoragePtr TableFunctionGenerateSeries::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +{ + if (const auto * function = ast_function->as()) + { + auto arguments = function->arguments->children; + + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + + UInt64 start = evaluateArgument(context, arguments[0]); + UInt64 stop = evaluateArgument(context, arguments[1]); + UInt64 interval = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; + if (start > stop) { + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0); + res->startup(); + return res; + } + + auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) / interval + 1, start, interval); + res->startup(); + return res; + } + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'limit' or 'offset, limit'.", getName()); +} + +UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const +{ + const auto & [field, type] = evaluateConstantExpression(argument, context); + + if (!isNativeNumber(type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} expression, must be numeric type", type->getName()); + + Field converted = convertFieldToType(field, DataTypeUInt64()); + if (converted.isNull()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); + + return converted.safeGet(); +} + + +} + +void registerTableFunctionGenerateSeries(TableFunctionFactory & factory) +{ + factory.registerFunction({.documentation = {}, .allow_readonly = true}); + // factory.registerFunction({.documentation = {}, .allow_readonly = true}); +} + +} diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 8c18c298f45..1631fa8e879 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -11,6 +11,7 @@ void registerTableFunctions() registerTableFunctionMerge(factory); registerTableFunctionRemote(factory); registerTableFunctionNumbers(factory); + registerTableFunctionGenerateSeries(factory); registerTableFunctionNull(factory); registerTableFunctionZeros(factory); registerTableFunctionExecutable(factory); diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index fae763e7dc8..111fbe8c22f 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -8,6 +8,7 @@ class TableFunctionFactory; void registerTableFunctionMerge(TableFunctionFactory & factory); void registerTableFunctionRemote(TableFunctionFactory & factory); void registerTableFunctionNumbers(TableFunctionFactory & factory); +void registerTableFunctionGenerateSeries(TableFunctionFactory & factory); void registerTableFunctionNull(TableFunctionFactory & factory); void registerTableFunctionZeros(TableFunctionFactory & factory); void registerTableFunctionExecutable(TableFunctionFactory & factory); From 3f0cfbd8c0816b007ff85b1a3997696ce5ed3214 Mon Sep 17 00:00:00 2001 From: Daniil Ivanik Date: Sat, 3 Feb 2024 19:46:00 +0100 Subject: [PATCH 011/332] Kek --- src/Common/iota.cpp | 29 ++ src/Common/iota.h | 9 + .../QueryPlan/ReadFromSystemNumbersStep.cpp | 281 ++++++++++-------- 3 files changed, 197 insertions(+), 122 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 98f18eb195b..532c4bde76d 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -27,10 +27,39 @@ void iota(T * begin, size_t count, T first_value) return iotaImpl(begin, count, first_value); } +MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(template void NO_INLINE), + iotaWithStepImpl, MULTITARGET_FUNCTION_BODY((T * begin, size_t count, T first_value, T step) /// NOLINT + { + for (size_t i = 0; i < count; i++) + *(begin + i) = static_cast(first_value + i * step); + }) +) + +template +void iota_with_step(T * begin, size_t count, T first_value, T step) +{ +#if USE_MULTITARGET_CODE + if (isArchSupported(TargetArch::AVX2)) + return iotaWithStepImplAVX2(begin, count, first_value, step); + + if (isArchSupported(TargetArch::SSE42)) + return iotaWithStepImplSSE42(begin, count, first_value, step); +#endif + return iotaWithStepImpl(begin, count, first_value, step); +} + template void iota(UInt8 * begin, size_t count, UInt8 first_value); template void iota(UInt32 * begin, size_t count, UInt32 first_value); template void iota(UInt64 * begin, size_t count, UInt64 first_value); #if defined(OS_DARWIN) template void iota(size_t * begin, size_t count, size_t first_value); #endif + +template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +#if defined(OS_DARWIN) +extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +#endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index 7910274d15d..f40cde9d5db 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -31,4 +31,13 @@ extern template void iota(UInt64 * begin, size_t count, UInt64 first_value); #if defined(OS_DARWIN) extern template void iota(size_t * begin, size_t count, size_t first_value); #endif + +template void iota_with_step(T * begin, size_t count, T first_value, T step); + +extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +#if defined(OS_DARWIN) +extern template void iota(size_t * begin, size_t count, size_t first_value, size_t step); +#endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 13a14ffb917..f85473e43c3 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace DB { @@ -30,9 +32,9 @@ public: : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) { } - String getName() const override { return "Numbers"; } + static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } protected: @@ -57,10 +59,7 @@ protected: ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * end = &vec[filtered_block_size]; - iota(pos, static_cast(end - pos), UInt64{0}); - for (UInt64 p = 0; p < filtered_block_size; p += 1) { - vec[p] = vec[p] * inner_step + first_element; - } + iota_with_step(pos, static_cast(end - pos), first_element, inner_step); next += step; @@ -77,28 +76,46 @@ private: UInt64 inner_remainder; }; - -[[maybe_unused]] UInt128 sizeOfRange(const Range & r) -{ - UInt128 size; - if (r.right.isPositiveInfinity()) - return static_cast(std::numeric_limits::max()) - r.left.get() + r.left_included; - - size = static_cast(r.right.get()) - r.left.get() + 1; - - if (!r.left_included) - size--; - - if (!r.right_included) - size--; - assert(size >= 0); - return size; +struct RangeWithStep { + Range range; + UInt64 step; }; -[[maybe_unused]] auto sizeOfRanges(const Ranges & rs) +using RangesWithStep = std::vector; + +std::optional stepped_range_from_range(const Range& r, UInt64 step, UInt64 remainder) { + UInt64 begin = (r.left.get() / step) * step; + if (begin > std::numeric_limits::max() - remainder) { + return std::nullopt; + } + begin += remainder; + while (begin <= r.left.get() - r.left_included) { + if (std::numeric_limits::max() - step < begin) { + return std::nullopt; + } + begin += step; + } + + LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + UInt128 right_edge = (r.right.get() + r.right_included); + if (begin >= right_edge) { + return std::nullopt; + } + return std::optional{RangeWithStep{Range(begin, true, static_cast(right_edge - 1), true), step}}; +} + +[[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) +{ + if (r.range.right.isPositiveInfinity()) + return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; + + return static_cast(r.range.right.get() - r.range.left.get()) / r.step + 1; +}; + +[[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; - for (const Range & r : rs) + for (const RangeWithStep & r : rs) { /// total_size will never overflow total_size += sizeOfRange(r); @@ -127,7 +144,7 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource(const Ranges & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) + [[maybe_unused]] NumbersRangedSource(const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) { } @@ -187,9 +204,9 @@ protected: if (ranges.empty()) return {}; - auto first_value = [](const Range & r) { return r.left.get() + (r.left_included ? 0 : 1); }; + auto first_value = [](const RangeWithStep & r) { return r.range.left.get() + (r.range.left_included ? 0 : 1); }; - auto last_value = [](const Range & r) { return r.right.get() - (r.right_included ? 0 : 1); }; + auto last_value = [](const RangeWithStep & r) { return r.range.right.get() - (r.range.right_included ? 0 : 1); }; /// Find the data range. /// If data left is small, shrink block size. @@ -215,31 +232,33 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(last_value(range)) - first_value(range) + 1 - cursor.offset_in_range; + : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; /// set value to block - auto set_value = [&pos](UInt128 & start_value, UInt128 & end_value) + auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { if (end_value > std::numeric_limits::max()) { - while (start_value < end_value) - *(pos++) = start_value++; + while (start_value < end_value) { + *(pos++) = start_value; + start_value += this->step; + } } else { auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = end_value_64 - start_value_64; - iota(pos, static_cast(size), start_value_64); + iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } }; if (can_provide > need) { - UInt64 start_value = first_value(range) + cursor.offset_in_range; + UInt64 start_value = first_value(range) + cursor.offset_in_range * step; /// end_value will never overflow - iota(pos, static_cast(need), start_value); + iota_with_step(pos, static_cast(need), start_value, step); pos += need; provided += need; @@ -248,8 +267,8 @@ protected: else if (can_provide == need) { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range; - UInt128 end_value = start_value + need; + UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 end_value = start_value + need * step; set_value(start_value, end_value); provided += need; @@ -259,8 +278,8 @@ protected: else { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range; - UInt128 end_value = start_value + can_provide; + UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 end_value = start_value + can_provide * step; set_value(start_value, end_value); provided += static_cast(can_provide); @@ -277,13 +296,15 @@ protected: private: /// The ranges is shared between all streams. - Ranges ranges; + RangesWithStep ranges; /// Ranges state shared between all streams, actually is the start of the ranges. RangesStatePtr ranges_state; /// Base block size, will shrink when data left is not enough. UInt64 base_block_size; + + UInt64 step; }; } @@ -304,7 +325,7 @@ namespace /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -[[maybe_unused]] void shrinkRanges(Ranges & ranges, size_t size) +[[maybe_unused]] void shrinkRanges(RangesWithStep & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -323,9 +344,9 @@ namespace else { auto & range = ranges[i]; - UInt64 right = range.left.get() + static_cast(size); - range.right = Field(right); - range.right_included = !range.left_included; + UInt64 right = range.range.left.get() + static_cast(size); + range.range.right = Field(right); + range.range.right_included = !range.range.left_included; last_range_idx = i; break; } @@ -393,101 +414,117 @@ Pipe ReadFromSystemNumbersStep::makePipe() num_streams = 1; /// Build rpn of query filters - // KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); + KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); Pipe pipe; Ranges ranges; - // if (condition.extractPlainRanges(ranges)) - // { - // /// Intersect ranges with table range - // std::optional table_range; - // std::optional overflowed_table_range; + if (condition.extractPlainRanges(ranges)) + { + LOG_DEBUG(&Poco::Logger::get("My logger"), "Use optimization"); + /// Intersect ranges with table range + std::optional table_range; + std::optional overflowed_table_range; - // if (numbers_storage.limit.has_value()) - // { - // if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); - // } - // /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) - // else - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); - // auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - // overflowed_table_range.emplace( - // FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); - // } - // } - // else - // { - // table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); - // } + if (numbers_storage.limit.has_value()) + { + if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) + { + table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + } + /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) + else + { + table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); + auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); + overflowed_table_range.emplace( + FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); + } + } + else + { + table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); + } + LOG_DEBUG(&Poco::Logger::get("My logger"), "Found table ranges"); - // Ranges intersected_ranges; - // for (auto & r : ranges) - // { - // auto intersected_range = table_range->intersectWith(r); - // if (intersected_range) - // intersected_ranges.push_back(*intersected_range); - // } - // /// intersection with overflowed_table_range goes back. - // if (overflowed_table_range.has_value()) - // { - // for (auto & r : ranges) - // { - // auto intersected_range = overflowed_table_range->intersectWith(r); - // if (intersected_range) - // intersected_ranges.push_back(*overflowed_table_range); - // } - // } + RangesWithStep intersected_ranges; + for (auto & r : ranges) + { + auto intersected_range = table_range->intersectWith(r); + if (intersected_range.has_value()) { + auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) { + intersected_ranges.push_back(*range_with_step); + } + } + } - // /// ranges is blank, return a source who has no data - // if (intersected_ranges.empty()) - // { - // pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); - // return pipe; - // } - // const auto & limit_length = limit_length_and_offset.first; - // const auto & limit_offset = limit_length_and_offset.second; - // /// If intersected ranges is limited or we can pushdown limit. - // if (!intersected_ranges.rbegin()->right.isPositiveInfinity() || should_pushdown_limit) - // { - // UInt128 total_size = sizeOfRanges(intersected_ranges); - // UInt128 query_limit = limit_length + limit_offset; + for (const auto& range : intersected_ranges) { + LOG_DEBUG(&Poco::Logger::get("Ranges"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); + // std::cout << + } + /// intersection with overflowed_table_range goes back. + if (overflowed_table_range.has_value()) + { + for (auto & r : ranges) + { + auto intersected_range = overflowed_table_range->intersectWith(r); + if (intersected_range) { + auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, static_cast((static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) % numbers_storage.step)); + if (range_with_step) { + intersected_ranges.push_back(*range_with_step); + } + } + } + } - // /// limit total_size by query_limit - // if (should_pushdown_limit && query_limit < total_size) - // { - // total_size = query_limit; - // /// We should shrink intersected_ranges for case: - // /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - // shrinkRanges(intersected_ranges, total_size); - // } + /// ranges is blank, return a source who has no data + if (intersected_ranges.empty()) + { + pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + return pipe; + } + const auto & limit_length = limit_length_and_offset.first; + const auto & limit_offset = limit_length_and_offset.second; - // checkLimits(size_t(total_size)); + /// If intersected ranges is limited or we can pushdown limit. + if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) + { + UInt128 total_size = sizeOfRanges(intersected_ranges); + UInt128 query_limit = limit_length + limit_offset; - // if (total_size / max_block_size < num_streams) - // num_streams = static_cast(total_size / max_block_size); + /// limit total_size by query_limit + if (should_pushdown_limit && query_limit < total_size) + { + total_size = query_limit; + /// We should shrink intersected_ranges for case: + /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + shrinkRanges(intersected_ranges, total_size); + } - // if (num_streams == 0) - // num_streams = 1; + checkLimits(size_t(total_size)); - // /// Ranges state, all streams will share the state. - // auto ranges_state = std::make_shared(); - // for (size_t i = 0; i < num_streams; ++i) - // { - // auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + if (total_size / max_block_size < num_streams) + num_streams = static_cast(total_size / max_block_size); - // if (i == 0) - // source->addTotalRowsApprox(total_size); + if (num_streams == 0) + num_streams = 1; - // pipe.addSource(std::move(source)); - // } - // return pipe; - // } - // } + /// Ranges state, all streams will share the state. + auto ranges_state = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + + if (i == 0) + source->addTotalRowsApprox(total_size); + + pipe.addSource(std::move(source)); + } + return pipe; + } + } /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) From 623b42574587073845a76a5e28a502a792ee6662 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 6 Feb 2024 21:34:09 +0000 Subject: [PATCH 012/332] Add feature with the right author name --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 175 +++++++++++++----- src/Storages/SelectQueryInfo.h | 2 +- src/Storages/System/StorageSystemNumbers.cpp | 8 +- src/Storages/System/StorageSystemNumbers.h | 15 +- src/Storages/System/attachSystemTables.cpp | 2 +- src/TableFunctions/CMakeLists.txt | 2 +- src/TableFunctions/ITableFunction.cpp | 2 +- ...es.cpp => TableFunctionGenerateSeries.cpp} | 55 ++++-- src/TableFunctions/TableFunctionNumbers.cpp | 45 +++-- .../02970_generate_series.reference | 28 +++ .../0_stateless/02970_generate_series.sql | 14 ++ 11 files changed, 254 insertions(+), 94 deletions(-) rename src/TableFunctions/{TableFunctionsGenerateSeries.cpp => TableFunctionGenerateSeries.cpp} (65%) create mode 100644 tests/queries/0_stateless/02970_generate_series.reference create mode 100644 tests/queries/0_stateless/02970_generate_series.sql diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index f85473e43c3..4b957778c43 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -28,32 +28,37 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string& column_name, UInt64 inner_step_) - : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), step(step_), inner_step(inner_step_), inner_remainder(offset_ % inner_step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string & column_name, UInt64 inner_step_) + : ISource(createHeader(column_name)) + , block_size(block_size_) + , next(offset_) + , step(step_) + , inner_step(inner_step_) + , inner_remainder(offset_ % inner_step_) { } String getName() const override { return "Numbers"; } - - static Block createHeader(const std::string& column_name) { return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; } + static Block createHeader(const std::string & column_name) + { + return {ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), column_name)}; + } protected: Chunk generate() override { - UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; - if (first_element < curr) { + if (first_element < curr) first_element += inner_step; - } UInt64 filtered_block_size = 0; - if (first_element - curr >= block_size) { + if (first_element - curr >= block_size) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, filtered_block_size}; } - if (first_element - curr < block_size) { + if (first_element - curr < block_size) filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; - } auto column = ColumnUInt64::create(filtered_block_size); ColumnUInt64::Container & vec = column->getData(); @@ -76,32 +81,37 @@ private: UInt64 inner_remainder; }; -struct RangeWithStep { +struct RangeWithStep +{ Range range; UInt64 step; }; using RangesWithStep = std::vector; -std::optional stepped_range_from_range(const Range& r, UInt64 step, UInt64 remainder) { - UInt64 begin = (r.left.get() / step) * step; - if (begin > std::numeric_limits::max() - remainder) { +std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) +{ + if ((r.right.get() == 0) && (!r.right_included)) + return std::nullopt; + UInt64 begin = (r.left.get() / step) * step; + if (begin > std::numeric_limits::max() - remainder) return std::nullopt; - } begin += remainder; - while (begin <= r.left.get() - r.left_included) { - if (std::numeric_limits::max() - step < begin) { + + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + while ((r.left_included <= r.left.get()) && (begin <= r.left.get() - r.left_included)) + { + if (std::numeric_limits::max() - step < begin) return std::nullopt; - } begin += step; } - LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); - UInt128 right_edge = (r.right.get() + r.right_included); - if (begin >= right_edge) { + // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); + if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; - } - return std::optional{RangeWithStep{Range(begin, true, static_cast(right_edge - 1), true), step}}; + UInt64 right_edge_included = r.right.get() - (1 - r.right_included); + return std::optional{RangeWithStep{Range(begin, true, right_edge_included, true), step}}; } [[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) @@ -144,8 +154,17 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource(const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, const std::string& column_name) - : ISource(NumbersSource::createHeader(column_name)), ranges(ranges_), ranges_state(ranges_state_), base_block_size(base_block_size_) + [[maybe_unused]] NumbersRangedSource( + const RangesWithStep & ranges_, + RangesStatePtr & ranges_state_, + UInt64 base_block_size_, + UInt64 step_, + const std::string & column_name) + : ISource(NumbersSource::createHeader(column_name)) + , ranges(ranges_) + , ranges_state(ranges_state_) + , base_block_size(base_block_size_) + , step(step_) { } @@ -158,6 +177,7 @@ protected: { std::lock_guard lock(ranges_state->mutex); + UInt64 need = base_block_size_; UInt64 size = 0; /// how many item found. @@ -196,6 +216,10 @@ protected: } ranges_state->pos = end; + + LOG_DEBUG(&Poco::Logger::get("Range borders"), "Begin: {} {}", start.offset_in_ranges, static_cast(start.offset_in_range)); + LOG_DEBUG(&Poco::Logger::get("Range borders"), "End: {} {}", end.offset_in_ranges, static_cast(end.offset_in_range)); + return size; } @@ -234,12 +258,19 @@ protected: ? end.offset_in_range - cursor.offset_in_range : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; + LOG_DEBUG( + &Poco::Logger::get("Generate"), + "Can Provide: {}, Block size: {}", + static_cast(can_provide), + static_cast(block_size)); + /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { if (end_value > std::numeric_limits::max()) { - while (start_value < end_value) { + while (start_value < end_value) + { *(pos++) = start_value; start_value += this->step; } @@ -248,7 +279,9 @@ protected: { auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); - auto size = end_value_64 - start_value_64; + auto size = (end_value_64 - start_value_64) / this->step; + LOG_DEBUG( + &Poco::Logger::get("Iota"), "Size: {}, Step: {}, Start: {}", static_cast(size), this->step, start_value_64); iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } @@ -374,7 +407,7 @@ ReadFromSystemNumbersStep::ReadFromSystemNumbersStep( , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context).expression} , max_block_size{max_block_size_} , num_streams{num_streams_} - , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) + , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) , should_pushdown_limit(shouldPushdownLimit(query_info, limit_length_and_offset.first)) , limit(query_info.limit) , storage_limits(query_info.storage_limits) @@ -410,14 +443,28 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto & numbers_storage = storage->as(); + LOG_DEBUG( + &Poco::Logger::get("Parameters"), + "Parameters: Limit: {}, Offset: {} Step: {}", + numbers_storage.limit.value(), + numbers_storage.offset, + numbers_storage.step); + if (!numbers_storage.multithreaded) num_streams = 1; + Pipe pipe; + Ranges ranges; + + if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) + { + pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); + return pipe; + } + /// Build rpn of query filters KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); - Pipe pipe; - Ranges ranges; if (condition.extractPlainRanges(ranges)) { @@ -430,14 +477,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() { if (std::numeric_limits::max() - numbers_storage.offset >= *(numbers_storage.limit)) { - table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); + table_range.emplace( + FieldRef(numbers_storage.offset), true, FieldRef(numbers_storage.offset + *(numbers_storage.limit)), false); } /// UInt64 overflow, for example: SELECT number FROM numbers(18446744073709551614, 5) else { table_range.emplace(FieldRef(numbers_storage.offset), true, std::numeric_limits::max(), true); auto overflow_end = UInt128(numbers_storage.offset) + UInt128(*numbers_storage.limit); - overflowed_table_range.emplace( + overflowed_table_range.emplace( FieldRef(UInt64(0)), true, FieldRef(UInt64(overflow_end - std::numeric_limits::max() - 1)), false); } } @@ -451,34 +499,59 @@ Pipe ReadFromSystemNumbersStep::makePipe() for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); - if (intersected_range.has_value()) { - auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); - if (range_with_step.has_value()) { + if (intersected_range.has_value()) + { + LOG_DEBUG( + &Poco::Logger::get("Ranges"), + "Ranges: {} {} {} {}", + intersected_range->left.get(), + intersected_range->right.get(), + intersected_range->left_included, + intersected_range->right_included); + auto range_with_step = stepped_range_from_range( + intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) + { + LOG_DEBUG( + &Poco::Logger::get("Ranges With Step"), + "Ranges: {} {} {} {} {}", + range_with_step->range.left.get(), + range_with_step->range.right.get(), + range_with_step->range.left_included, + range_with_step->range.right_included, + range_with_step->step); intersected_ranges.push_back(*range_with_step); } } } - for (const auto& range : intersected_ranges) { - LOG_DEBUG(&Poco::Logger::get("Ranges"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); - // std::cout << - } /// intersection with overflowed_table_range goes back. if (overflowed_table_range.has_value()) { for (auto & r : ranges) { auto intersected_range = overflowed_table_range->intersectWith(r); - if (intersected_range) { - auto range_with_step = stepped_range_from_range(intersected_range.value(), numbers_storage.step, static_cast((static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) % numbers_storage.step)); - if (range_with_step) { + if (intersected_range) + { + auto range_with_step = stepped_range_from_range( + intersected_range.value(), + numbers_storage.step, + static_cast( + (static_cast(numbers_storage.offset) + std::numeric_limits::max() + 1) + % numbers_storage.step)); + if (range_with_step) intersected_ranges.push_back(*range_with_step); - } } } } + // for (const auto& range : intersected_ranges) + // { + // LOG_DEBUG(&Poco::Logger::get("Ranges with step"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); + // // std::cout << + // } + /// ranges is blank, return a source who has no data if (intersected_ranges.empty()) { @@ -492,6 +565,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); + LOG_DEBUG(&Poco::Logger::get("Total_Size"), "Total Size: {}", static_cast(total_size)); UInt128 query_limit = limit_length + limit_offset; /// limit total_size by query_limit @@ -515,7 +589,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto ranges_state = std::make_shared(); for (size_t i = 0; i < num_streams; ++i) { - auto source = std::make_shared(intersected_ranges, ranges_state, max_block_size, numbers_storage.column_name); + auto source = std::make_shared( + intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); if (i == 0) source->addTotalRowsApprox(total_size); @@ -529,12 +604,16 @@ Pipe ReadFromSystemNumbersStep::makePipe() /// Fall back to NumbersSource for (size_t i = 0; i < num_streams; ++i) { - auto source - = std::make_shared(max_block_size, numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, numbers_storage.step); + auto source = std::make_shared( + max_block_size, + numbers_storage.offset + i * max_block_size, + num_streams * max_block_size, + numbers_storage.column_name, + numbers_storage.step); if (numbers_storage.limit && i == 0) { - auto rows_appr = *(numbers_storage.limit); + auto rows_appr = (*numbers_storage.limit - 1) / numbers_storage.step + 1; if (limit > 0 && limit < rows_appr) rows_appr = limit; source->addTotalRowsApprox(rows_appr); @@ -546,7 +625,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (numbers_storage.limit) { size_t i = 0; - auto storage_limit = *(numbers_storage.limit); + auto storage_limit = (*numbers_storage.limit - 1) / numbers_storage.step + 1; /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. pipe.addSimpleTransform( [&](const Block & header) diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 2b4afaa6345..662a5c0ef5a 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -229,4 +229,4 @@ struct SelectQueryInfo bool isFinal() const; }; - } +} diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index cd7207917a9..4c319ec7105 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -16,7 +16,13 @@ namespace DB { -StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name_, std::optional limit_, UInt64 offset_, UInt64 step_) +StorageSystemNumbers::StorageSystemNumbers( + const StorageID & table_id, + bool multithreaded_, + const std::string & column_name_, + std::optional limit_, + UInt64 offset_, + UInt64 step_) : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index ffe87b8ad14..9663ee25251 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,11 +10,17 @@ namespace DB class Context; -class StorageSystemNumbers final : public IStorage +class StorageSystemNumbers final : public IStorage { public: /// Otherwise, streams concurrently increment atomic. - StorageSystemNumbers(const StorageID & table_id, bool multithreaded_, const std::string& column_name, std::optional limit_ = std::nullopt, UInt64 offset_ = 0, UInt64 step_ = 1); + StorageSystemNumbers( + const StorageID & table_id, + bool multithreaded_, + const std::string & column_name, + std::optional limit_ = std::nullopt, + UInt64 offset_ = 0, + UInt64 step_ = 1); std::string getName() const override { return "SystemNumbers"; } @@ -30,7 +36,6 @@ public: bool hasEvenlyDistributedRead() const override { return true; } bool isSystemStorage() const override { return true; } - bool supportsTransactions() const override { return true; } private: @@ -38,11 +43,9 @@ private: bool multithreaded; std::optional limit; - UInt64 offset; + UInt64 offset;` std::string column_name; - UInt64 step; - }; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index ddd89709b6a..9eacb07bd8d 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -120,7 +120,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attach(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); attach(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); attach(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); - // attach(context, system_database, "generate_series", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", false, "generate_series"); + attach(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attach(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attach(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index c5c2a660935..770990cc405 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -29,7 +29,7 @@ if (TARGET ch_contrib::azure_sdk) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::azure_sdk) endif () -if (TARGET ch_co`trib::simdjson) +if (TARGET ch_contrib::simdjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::simdjson) endif () diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index c854b6b0f9c..137e1dc27fe 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -5,7 +5,7 @@ #include #include #include -#include + namespace ProfileEvents { diff --git a/src/TableFunctions/TableFunctionsGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp similarity index 65% rename from src/TableFunctions/TableFunctionsGenerateSeries.cpp rename to src/TableFunctions/TableFunctionGenerateSeries.cpp index 3941f1eadb2..88d7b0d1a71 100644 --- a/src/TableFunctions/TableFunctionsGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -1,13 +1,13 @@ +#include +#include +#include +#include +#include +#include #include #include -#include -#include #include -#include -#include -#include -#include -#include +#include #include "registerTableFunctions.h" @@ -18,6 +18,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INVALID_SETTING_VALUE; } namespace @@ -33,8 +34,14 @@ public: static constexpr auto name = "generate_series"; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } + private: - StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + StoragePtr executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription cached_columns, + bool is_insert_query) const override; const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; @@ -48,25 +55,31 @@ ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextP return ColumnsDescription{{{"generate_series", std::make_shared()}}}; } -StoragePtr TableFunctionGenerateSeries::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +StoragePtr TableFunctionGenerateSeries::executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const { if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; if (arguments.size() != 2 && arguments.size() != 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); UInt64 start = evaluateArgument(context, arguments[0]); UInt64 stop = evaluateArgument(context, arguments[1]); - UInt64 interval = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; - if (start > stop) { - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0); - res->startup(); - return res; - } - - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) / interval + 1, start, interval); + UInt64 step = (arguments.size() == 3) ? evaluateArgument(context, arguments[2]) : UInt64{1}; + if (step == UInt64{0}) + throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Table function '{}' requires step to be a positive number", getName()); + auto res = (start > stop) + ? std::make_shared( + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 0) + : std::make_shared( + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) + 1, start, step); res->startup(); return res; } @@ -82,8 +95,10 @@ UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr Field converted = convertFieldToType(field, DataTypeUInt64()); if (converted.isNull()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", - applyVisitor(FieldVisitorToString(), field)); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); return converted.safeGet(); } diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 71a9ba097c6..bcda8dc6a5e 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -1,13 +1,13 @@ +#include +#include +#include +#include +#include +#include #include #include -#include -#include #include -#include -#include -#include -#include -#include +#include #include "registerTableFunctions.h" @@ -16,8 +16,8 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace @@ -34,8 +34,14 @@ public: static constexpr auto name = multithreaded ? "numbers_mt" : "numbers"; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } + private: - StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool is_insert_query) const override; + StoragePtr executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription cached_columns, + bool is_insert_query) const override; const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(ContextPtr context, ASTPtr & argument) const; @@ -51,19 +57,26 @@ ColumnsDescription TableFunctionNumbers::getActualTableStructure( } template -StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const +StoragePtr TableFunctionNumbers::executeImpl( + const ASTPtr & ast_function, + ContextPtr context, + const std::string & table_name, + ColumnsDescription /*cached_columns*/, + bool /*is_insert_query*/) const { if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; if (arguments.size() != 1 && arguments.size() != 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); - auto res = std::make_shared(StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); + auto res = std::make_shared( + StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); res->startup(); return res; } @@ -80,8 +93,10 @@ UInt64 TableFunctionNumbers::evaluateArgument(ContextPtr context, Field converted = convertFieldToType(field, DataTypeUInt64()); if (converted.isNull()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", - applyVisitor(FieldVisitorToString(), field)); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The value {} is not representable as UInt64", + applyVisitor(FieldVisitorToString(), field)); return converted.safeGet(); } diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference new file mode 100644 index 00000000000..9e6f1db911e --- /dev/null +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -0,0 +1,28 @@ +0 +1 +4 +8 +500000001 +50000000 +100000001 +0 +10 +13 +16 +19 +7 +17 +27 +37 +47 +57 +67 +77 +17 +22 +27 +32 +37 +42 +47 +52 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql new file mode 100644 index 00000000000..045f584a622 --- /dev/null +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -0,0 +1,14 @@ +SELECT count() FROM generate_series(5, 4); +SELECT count() FROM generate_series(0, 0); +SELECT count() FROM generate_series(10, 20, 3); +SELECT count() FROM generate_series(7, 77, 10); +SELECT count() FROM generate_series(0, 1000000000, 2); +SELECT count() FROM generate_series(0, 999999999, 20); +SELECT count() FROM generate_series(0, 1000000000, 2) WHERE generate_series % 5 == 0; + +SELECT * FROM generate_series(5, 4); +SELECT * FROM generate_series(0, 0); +SELECT * FROM generate_series(10, 20, 3); +SELECT * FROM generate_series(7, 77, 10); +SELECT * FROM generate_series(7, 52, 5) WHERE generate_series >= 13; + From 145e425ddd5707a5852dd3c6ac2672ccbd68e2bd Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 7 Feb 2024 15:29:45 +0000 Subject: [PATCH 013/332] Added Documentation --- .../table-functions/generate_series.md | 25 +++++++++ .../QueryPlan/ReadFromSystemNumbersStep.cpp | 53 ------------------- src/Storages/System/StorageSystemNumbers.h | 2 +- 3 files changed, 26 insertions(+), 54 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/generate_series.md diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md new file mode 100644 index 00000000000..de34e10ac76 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -0,0 +1,25 @@ +--- +slug: /en/sql-reference/table-functions/generate_series +sidebar_position: ? +sidebar_label: generate_series +--- + +# generate_series + +`generate_series(START, STOP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively. + +`generate_series(START, STOP, STEP)` - Returns a table with the single ‘generate_series’ column (UInt64) that contains integers from start to stop inclusively with spacing between values given by STEP. + +The following queries return tables with the same content but different column names: + +``` sql +SELECT * FROM numbers(10, 5); +SELECT * FROM generate_series(10, 14); +``` + +And the following queries return tables with the same content but different column names (but the second option is more efficient): + +``` sql +SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); +SELECT * FROM generate_series(10, 20, 3) ; +``` \ No newline at end of file diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 4b957778c43..3bb2e0cd69d 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -98,8 +98,6 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return std::nullopt; begin += remainder; - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); while ((r.left_included <= r.left.get()) && (begin <= r.left.get() - r.left_included)) { if (std::numeric_limits::max() - step < begin) @@ -107,7 +105,6 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st begin += step; } - // LOG_DEBUG(&Poco::Logger::get("stepped_range_from_range"), "Begin: {}", begin); if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); @@ -217,9 +214,6 @@ protected: ranges_state->pos = end; - LOG_DEBUG(&Poco::Logger::get("Range borders"), "Begin: {} {}", start.offset_in_ranges, static_cast(start.offset_in_range)); - LOG_DEBUG(&Poco::Logger::get("Range borders"), "End: {} {}", end.offset_in_ranges, static_cast(end.offset_in_range)); - return size; } @@ -258,12 +252,6 @@ protected: ? end.offset_in_range - cursor.offset_in_range : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; - LOG_DEBUG( - &Poco::Logger::get("Generate"), - "Can Provide: {}, Block size: {}", - static_cast(can_provide), - static_cast(block_size)); - /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) { @@ -280,8 +268,6 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - LOG_DEBUG( - &Poco::Logger::get("Iota"), "Size: {}, Step: {}, Start: {}", static_cast(size), this->step, start_value_64); iota_with_step(pos, static_cast(size), start_value_64, step); pos += size; } @@ -443,13 +429,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto & numbers_storage = storage->as(); - LOG_DEBUG( - &Poco::Logger::get("Parameters"), - "Parameters: Limit: {}, Offset: {} Step: {}", - numbers_storage.limit.value(), - numbers_storage.offset, - numbers_storage.step); - if (!numbers_storage.multithreaded) num_streams = 1; @@ -468,7 +447,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (condition.extractPlainRanges(ranges)) { - LOG_DEBUG(&Poco::Logger::get("My logger"), "Use optimization"); /// Intersect ranges with table range std::optional table_range; std::optional overflowed_table_range; @@ -493,36 +471,11 @@ Pipe ReadFromSystemNumbersStep::makePipe() { table_range.emplace(FieldRef(numbers_storage.offset), true, FieldRef(std::numeric_limits::max()), true); } - LOG_DEBUG(&Poco::Logger::get("My logger"), "Found table ranges"); RangesWithStep intersected_ranges; for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); - if (intersected_range.has_value()) - { - LOG_DEBUG( - &Poco::Logger::get("Ranges"), - "Ranges: {} {} {} {}", - intersected_range->left.get(), - intersected_range->right.get(), - intersected_range->left_included, - intersected_range->right_included); - auto range_with_step = stepped_range_from_range( - intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); - if (range_with_step.has_value()) - { - LOG_DEBUG( - &Poco::Logger::get("Ranges With Step"), - "Ranges: {} {} {} {} {}", - range_with_step->range.left.get(), - range_with_step->range.right.get(), - range_with_step->range.left_included, - range_with_step->range.right_included, - range_with_step->step); - intersected_ranges.push_back(*range_with_step); - } - } } @@ -546,11 +499,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() } } - // for (const auto& range : intersected_ranges) - // { - // LOG_DEBUG(&Poco::Logger::get("Ranges with step"), "Left: {}; Right {}, LI: {}, RI: {}, Step: {}", range.range.left.get(), range.range.right.get(), range.range.left_included, range.range.right_included, range.step); - // // std::cout << - // } /// ranges is blank, return a source who has no data if (intersected_ranges.empty()) @@ -565,7 +513,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); - LOG_DEBUG(&Poco::Logger::get("Total_Size"), "Total Size: {}", static_cast(total_size)); UInt128 query_limit = limit_length + limit_offset; /// limit total_size by query_limit diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index 9663ee25251..298721984b8 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -43,7 +43,7 @@ private: bool multithreaded; std::optional limit; - UInt64 offset;` + UInt64 offset; std::string column_name; UInt64 step; }; From 03aaedace439f5db6d9a6aaf91a1b2f978b0f6a9 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 12:05:01 +0000 Subject: [PATCH 014/332] Fix bug --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3bb2e0cd69d..bc14547889b 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -435,6 +435,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; + // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {}", numbers_storage.step, numbers_storage.limit.value(), numbers_storage.offset); + if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); @@ -476,6 +478,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() for (auto & r : ranges) { auto intersected_range = table_range->intersectWith(r); + if (intersected_range.has_value()) + { + auto range_with_step = stepped_range_from_range( + intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + if (range_with_step.has_value()) + { + intersected_ranges.push_back(*range_with_step); + } + } } From 1b2f23247b7f115ba92b9908d224d4e78e8649f4 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 12:28:54 +0000 Subject: [PATCH 015/332] Change documentation --- docs/en/sql-reference/table-functions/generate_series.md | 2 +- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index de34e10ac76..c5d29369627 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -1,6 +1,6 @@ --- slug: /en/sql-reference/table-functions/generate_series -sidebar_position: ? +sidebar_position: 146 sidebar_label: generate_series --- diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index bc14547889b..ab2f726aeb5 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -483,9 +483,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto range_with_step = stepped_range_from_range( intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (range_with_step.has_value()) - { intersected_ranges.push_back(*range_with_step); - } } } From 79f91003538a71014eb035dca024285f2fbba7d5 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 14:17:25 +0000 Subject: [PATCH 016/332] To pull --- src/TableFunctions/TableFunctionGenerateSeries.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp index 88d7b0d1a71..65c4c4915c2 100644 --- a/src/TableFunctions/TableFunctionGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -77,7 +77,7 @@ StoragePtr TableFunctionGenerateSeries::executeImpl( throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Table function '{}' requires step to be a positive number", getName()); auto res = (start > stop) ? std::make_shared( - StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 0) + StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, 0, 0, 1) : std::make_shared( StorageID(getDatabaseName(), table_name), false, std::string{"generate_series"}, (stop - start) + 1, start, step); res->startup(); From f7dbcdd7e7e00d4fb6d30a02ebcb4a3befcd3190 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 9 Feb 2024 18:12:24 +0000 Subject: [PATCH 017/332] Made refactoring --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 41 ++++++++++++------- src/Storages/System/StorageSystemNumbers.h | 28 +++++++++++++ 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3a905a56aa1..2488fa37643 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -9,10 +9,12 @@ #include #include #include +#include #include #include #include +#include "base/types.h" namespace DB { @@ -28,13 +30,13 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_, const std::string & column_name, UInt64 inner_step_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_, UInt64 remainder_) : ISource(createHeader(column_name)) , block_size(block_size_) , next(offset_) + , chunk_step(chunk_step_) , step(step_) - , inner_step(inner_step_) - , inner_remainder(offset_ % inner_step_) + , remainder(remainder_) { } String getName() const override { return "Numbers"; } @@ -48,25 +50,33 @@ protected: Chunk generate() override { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. - UInt64 first_element = (curr / inner_step) * inner_step + inner_remainder; - if (first_element < curr) - first_element += inner_step; - UInt64 filtered_block_size = 0; + UInt64 first_element = (curr / step) * step; + if (first_element > std::numeric_limits::max() - remainder) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, 0}; + } + first_element += remainder; + if (first_element < curr) { + if (first_element > std::numeric_limits::max() - step) { + auto column = ColumnUInt64::create(0); + return {Columns{std::move(column)}, 0}; + } + first_element += step; + } if (first_element - curr >= block_size) { auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, filtered_block_size}; + return {Columns{std::move(column)}, 0}; } - if (first_element - curr < block_size) - filtered_block_size = (block_size - (first_element - curr) - 1) / inner_step + 1; + UInt64 filtered_block_size = (block_size - (first_element - curr) - 1) / step + 1; auto column = ColumnUInt64::create(filtered_block_size); ColumnUInt64::Container & vec = column->getData(); UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * end = &vec[filtered_block_size]; - iota_with_step(pos, static_cast(end - pos), first_element, inner_step); + iota_with_step(pos, static_cast(end - pos), first_element, step); - next += step; + next += chunk_step; progress(column->size(), column->byteSize()); @@ -76,9 +86,9 @@ protected: private: UInt64 block_size; UInt64 next; + UInt64 chunk_step; UInt64 step; - UInt64 inner_step; - UInt64 inner_remainder; + UInt64 remainder; }; struct RangeWithStep @@ -565,7 +575,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, - numbers_storage.step); + numbers_storage.step, + numbers_storage.offset % numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index 298721984b8..c698bae4393 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -10,6 +10,34 @@ namespace DB class Context; +/** Implements a table engine for the system table "numbers". + * The table contains the only column number UInt64. + * From this table, you can read all natural numbers, starting from 0 (to 2^64 - 1, and then again). + * + * You could also specify a limit (how many numbers to give). + * + * How to generate numbers? + * + * 1. First try a smart fashion: + * + * In this fashion we try to push filters and limit down to scanning. + * Firstly extract plain ranges(no overlapping and ordered) by filter expressions. + * + * For example: + * where (numbers > 1 and numbers < 3) or (numbers in (4, 6)) or (numbers > 7 and numbers < 9) + * + * We will get ranges + * (1, 3), [4, 4], [6, 6], (7, 9) + * + * Then split the ranges evenly to one or multi-streams. With this way we will get result without large scanning. + * + * 2. If fail to extract plain ranges, fall back to ordinary scanning. + * + * If multithreaded is specified, numbers will be generated in several streams + * (and result could be out of order). If both multithreaded and limit are specified, + * the table could give you not exactly 1..limit range, but some arbitrary 'limit' numbers. + */ + class StorageSystemNumbers final : public IStorage { public: From 696609e7d562d15cfc7a6ffa776785444a97c2e7 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 19:59:57 +0000 Subject: [PATCH 018/332] Kek --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 2488fa37643..dc6aebc69c1 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -14,6 +14,7 @@ #include #include +#include "Core/Types.h" #include "base/types.h" namespace DB @@ -51,13 +52,16 @@ protected: { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) { + if (first_element > std::numeric_limits::max() - remainder) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } first_element += remainder; - if (first_element < curr) { - if (first_element > std::numeric_limits::max() - step) { + if (first_element < curr) + { + if (first_element > std::numeric_limits::max() - step) + { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } @@ -101,6 +105,8 @@ using RangesWithStep = std::vector; std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) { + // LOG_DEBUG(&Poco::Logger::get("Stepped from range"), + // "stepped from range"); if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; UInt64 begin = (r.left.get() / step) * step; @@ -126,7 +132,11 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if (r.range.right.isPositiveInfinity()) return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; - return static_cast(r.range.right.get() - r.range.left.get()) / r.step + 1; + UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; + if (r.range.right_included && (r.range.right.get() % r.step == 0)) { + ++size; + } + return size; }; [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) @@ -173,6 +183,17 @@ public: , base_block_size(base_block_size_) , step(step_) { + // for (const auto& range_with_step : ranges_) { + // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), + // // "Ranges: {} {} {} {} {}", + // // range_with_step.range.left.get(), + // // range_with_step.range.right.get(), + // // range_with_step.range.left_included, + // // range_with_step.range.right_included, + // // range_with_step.step); + // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), + // // "Step: {}", step); + // } } String getName() const override { return "NumbersRange"; } @@ -241,6 +262,8 @@ protected: RangesPos start, end; auto block_size = findRanges(start, end, base_block_size); + // LOG_DEBUG(&Poco::Logger::get("Found range"), "Evth: {} {} {} {} {} {}", start.offset_in_ranges, static_cast(start.offset_in_range), end.offset_in_ranges, static_cast(end.offset_in_range), base_block_size, block_size); + if (!block_size) return {}; @@ -256,6 +279,11 @@ protected: while (block_size - provided != 0) { UInt64 need = block_size - provided; + // LOG_DEBUG(&Poco::Logger::get("Indices:"), + // "Indices: {} {}, provided: {}", + // ranges.size(), + // cursor.offset_in_ranges, + // provided); auto & range = ranges[cursor.offset_in_ranges]; UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges @@ -445,13 +473,15 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; - // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {}", numbers_storage.step, numbers_storage.limit.value(), numbers_storage.offset); + + // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {} {}", numbers_storage.step, numbers_storage.offset, numbers_storage.limit.has_value(), numbers_storage.limit.has_value() ? numbers_storage.limit.value() : UInt64{0}); if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); return pipe; } + chassert(numbers_storage.step != UInt64{0}); /// Build rpn of query filters KeyCondition condition(buildFilterDAG(), context, column_names, key_expression); @@ -575,7 +605,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() numbers_storage.offset + i * max_block_size, num_streams * max_block_size, numbers_storage.column_name, - numbers_storage.step, + numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (numbers_storage.limit && i == 0) From 3ec9f3c4c89dec2f1971979d7d3ae406c1ecd938 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 20:06:52 +0000 Subject: [PATCH 019/332] Check foormattign --- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index dc6aebc69c1..1e6b539ee2e 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -52,15 +52,15 @@ protected: { UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) + if (first_element > std::numeric_limits::max() - remainder) { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; } first_element += remainder; - if (first_element < curr) + if (first_element < curr) { - if (first_element > std::numeric_limits::max() - step) + if (first_element > std::numeric_limits::max() - step) { auto column = ColumnUInt64::create(0); return {Columns{std::move(column)}, 0}; @@ -133,9 +133,8 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; - if (r.range.right_included && (r.range.right.get() % r.step == 0)) { + if (r.range.right_included && (r.range.right.get() % r.step == 0)) ++size; - } return size; }; From d0456980991c45935fd316ca7dc2bd61cf45e5b9 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 23:04:52 +0000 Subject: [PATCH 020/332] It seems to work --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 31 ++++++------------- ...ble_functions_must_be_documented.reference | 1 + 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 1e6b539ee2e..2217e426b02 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -97,7 +97,8 @@ private: struct RangeWithStep { - Range range; + UInt64 left; + UInt64 right; UInt64 step; }; @@ -124,18 +125,12 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); - return std::optional{RangeWithStep{Range(begin, true, right_edge_included, true), step}}; + return std::optional{RangeWithStep{begin, right_edge_included, step}}; } [[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) { - if (r.range.right.isPositiveInfinity()) - return static_cast(std::numeric_limits::max() - r.range.left.get()) / r.step + r.range.left_included; - - UInt128 size = static_cast(r.range.right.get() - r.range.left.get()) / r.step; - if (r.range.right_included && (r.range.right.get() % r.step == 0)) - ++size; - return size; + return static_cast(r.right - r.left) / r.step + 1; }; [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) @@ -252,10 +247,6 @@ protected: if (ranges.empty()) return {}; - auto first_value = [](const RangeWithStep & r) { return r.range.left.get() + (r.range.left_included ? 0 : 1); }; - - auto last_value = [](const RangeWithStep & r) { return r.range.right.get() - (r.range.right_included ? 0 : 1); }; - /// Find the data range. /// If data left is small, shrink block size. RangesPos start, end; @@ -287,7 +278,7 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(last_value(range) - first_value(range)) / range.step + 1 - cursor.offset_in_range; + : static_cast(range.right - range.left) / range.step + 1 - cursor.offset_in_range; /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) @@ -312,7 +303,7 @@ protected: if (can_provide > need) { - UInt64 start_value = first_value(range) + cursor.offset_in_range * step; + UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow iota_with_step(pos, static_cast(need), start_value, step); pos += need; @@ -323,7 +314,7 @@ protected: else if (can_provide == need) { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 start_value = static_cast(range.left) + cursor.offset_in_range * step; UInt128 end_value = start_value + need * step; set_value(start_value, end_value); @@ -334,7 +325,7 @@ protected: else { /// to avoid UInt64 overflow - UInt128 start_value = static_cast(first_value(range)) + cursor.offset_in_range * step; + UInt128 start_value = static_cast(range.left) + cursor.offset_in_range * step; UInt128 end_value = start_value + can_provide * step; set_value(start_value, end_value); @@ -400,9 +391,7 @@ namespace else { auto & range = ranges[i]; - UInt64 right = range.range.left.get() + static_cast(size); - range.range.right = Field(right); - range.range.right_included = !range.range.left_included; + range.right = range.left + static_cast(size) * range.step - 1; last_range_idx = i; break; } @@ -558,7 +547,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_offset = limit_length_and_offset.second; /// If intersected ranges is limited or we can pushdown limit. - if (!intersected_ranges.rbegin()->range.right.isPositiveInfinity() || should_pushdown_limit) + if (should_pushdown_limit) { UInt128 total_size = sizeOfRanges(intersected_ranges); UInt128 query_limit = limit_length + limit_offset; diff --git a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference index e4040a2d371..1e4f21a6722 100644 --- a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference @@ -4,6 +4,7 @@ dictionary executable file generateRandom +generate_series input jdbc merge From 789d3c699c77d7a39f42281d9dc0c61010471242 Mon Sep 17 00:00:00 2001 From: divanik Date: Sat, 10 Feb 2024 23:49:58 +0000 Subject: [PATCH 021/332] Remove bug for mt --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 2217e426b02..3656a6d31ee 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -547,42 +547,39 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_offset = limit_length_and_offset.second; /// If intersected ranges is limited or we can pushdown limit. - if (should_pushdown_limit) + UInt128 total_size = sizeOfRanges(intersected_ranges); + UInt128 query_limit = limit_length + limit_offset; + + /// limit total_size by query_limit + if (should_pushdown_limit && query_limit < total_size) { - UInt128 total_size = sizeOfRanges(intersected_ranges); - UInt128 query_limit = limit_length + limit_offset; - - /// limit total_size by query_limit - if (should_pushdown_limit && query_limit < total_size) - { - total_size = query_limit; - /// We should shrink intersected_ranges for case: - /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 - shrinkRanges(intersected_ranges, total_size); - } - - checkLimits(size_t(total_size)); - - if (total_size / max_block_size < num_streams) - num_streams = static_cast(total_size / max_block_size); - - if (num_streams == 0) - num_streams = 1; - - /// Ranges state, all streams will share the state. - auto ranges_state = std::make_shared(); - for (size_t i = 0; i < num_streams; ++i) - { - auto source = std::make_shared( - intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); - - if (i == 0) - source->addTotalRowsApprox(total_size); - - pipe.addSource(std::move(source)); - } - return pipe; + total_size = query_limit; + /// We should shrink intersected_ranges for case: + /// intersected_ranges: [1, 4], [7, 100]; query_limit: 2 + shrinkRanges(intersected_ranges, total_size); } + + checkLimits(size_t(total_size)); + + if (total_size / max_block_size < num_streams) + num_streams = static_cast(total_size / max_block_size); + + if (num_streams == 0) + num_streams = 1; + + /// Ranges state, all streams will share the state. + auto ranges_state = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared( + intersected_ranges, ranges_state, max_block_size, numbers_storage.step, numbers_storage.column_name); + + if (i == 0) + source->addTotalRowsApprox(total_size); + + pipe.addSource(std::move(source)); + } + return pipe; } /// Fall back to NumbersSource From 0f84f68da77663e2adcce800cceefff5ab019b58 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 13 Feb 2024 09:59:39 +0000 Subject: [PATCH 022/332] Simplified code --- src/Common/iota.cpp | 2 +- src/Common/iota.h | 2 +- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 65 +++++-------------- .../02970_generate_series.reference | 1 - .../0_stateless/02970_generate_series.sql | 1 - 5 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 532c4bde76d..86c9e04bb06 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -60,6 +60,6 @@ template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UIn template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index f40cde9d5db..8fa18be9769 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -38,6 +38,6 @@ extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_val extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota(size_t * begin, size_t count, size_t first_value, size_t step); +extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 3656a6d31ee..d69e2b6ca5a 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -15,6 +15,7 @@ #include #include "Core/Types.h" +#include "base/Decimal_fwd.h" #include "base/types.h" namespace DB @@ -31,13 +32,12 @@ namespace class NumbersSource : public ISource { public: - NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_, UInt64 remainder_) + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_) : ISource(createHeader(column_name)) , block_size(block_size_) , next(offset_) , chunk_step(chunk_step_) , step(step_) - , remainder(remainder_) { } String getName() const override { return "Numbers"; } @@ -50,41 +50,19 @@ public: protected: Chunk generate() override { - UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. - UInt64 first_element = (curr / step) * step; - if (first_element > std::numeric_limits::max() - remainder) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - first_element += remainder; - if (first_element < curr) - { - if (first_element > std::numeric_limits::max() - step) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - first_element += step; - } - if (first_element - curr >= block_size) - { - auto column = ColumnUInt64::create(0); - return {Columns{std::move(column)}, 0}; - } - UInt64 filtered_block_size = (block_size - (first_element - curr) - 1) / step + 1; - - auto column = ColumnUInt64::create(filtered_block_size); + auto column = ColumnUInt64::create(block_size); ColumnUInt64::Container & vec = column->getData(); + + UInt64 curr = next; /// The local variable for some reason works faster (>20%) than member of class. UInt64 * pos = vec.data(); /// This also accelerates the code. - UInt64 * end = &vec[filtered_block_size]; - iota_with_step(pos, static_cast(end - pos), first_element, step); + UInt64 * end = &vec[block_size]; + iota_with_step(pos, static_cast(end - pos), curr, step); next += chunk_step; progress(column->size(), column->byteSize()); - return {Columns{std::move(column)}, filtered_block_size}; + return {Columns{std::move(column)}, block_size}; } private: @@ -92,14 +70,13 @@ private: UInt64 next; UInt64 chunk_step; UInt64 step; - UInt64 remainder; }; struct RangeWithStep { UInt64 left; - UInt64 right; UInt64 step; + UInt128 size; }; using RangesWithStep = std::vector; @@ -125,21 +102,16 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st if ((begin >= r.right_included) && (begin - r.right_included >= r.right.get())) return std::nullopt; UInt64 right_edge_included = r.right.get() - (1 - r.right_included); - return std::optional{RangeWithStep{begin, right_edge_included, step}}; + return std::optional{RangeWithStep{begin, step, static_cast(right_edge_included - begin) / step + 1}}; } -[[maybe_unused]] UInt128 sizeOfRange(const RangeWithStep & r) -{ - return static_cast(r.right - r.left) / r.step + 1; -}; - [[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; for (const RangeWithStep & r : rs) { /// total_size will never overflow - total_size += sizeOfRange(r); + total_size += r.size; } return total_size; }; @@ -211,7 +183,7 @@ protected: while (need != 0) { UInt128 can_provide = end.offset_in_ranges == ranges.size() ? static_cast(0) - : sizeOfRange(ranges[end.offset_in_ranges]) - end.offset_in_range; + : ranges[end.offset_in_ranges].size - end.offset_in_range; if (can_provide == 0) break; @@ -278,7 +250,7 @@ protected: UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges ? end.offset_in_range - cursor.offset_in_range - : static_cast(range.right - range.left) / range.step + 1 - cursor.offset_in_range; + : range.size - cursor.offset_in_range; /// set value to block auto set_value = [&pos, this](UInt128 & start_value, UInt128 & end_value) @@ -377,7 +349,7 @@ namespace size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) { - auto range_size = sizeOfRange(ranges[i]); + auto range_size = ranges[i].size; if (range_size < size) { size -= static_cast(range_size); @@ -391,7 +363,7 @@ namespace else { auto & range = ranges[i]; - range.right = range.left + static_cast(size) * range.step - 1; + range.size = static_cast(size); last_range_idx = i; break; } @@ -587,11 +559,10 @@ Pipe ReadFromSystemNumbersStep::makePipe() { auto source = std::make_shared( max_block_size, - numbers_storage.offset + i * max_block_size, - num_streams * max_block_size, + numbers_storage.offset + i * max_block_size * numbers_storage.step, + num_streams * max_block_size * numbers_storage.step, numbers_storage.column_name, - numbers_storage.step, - numbers_storage.offset % numbers_storage.step); + numbers_storage.step); if (numbers_storage.limit && i == 0) { diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference index 9e6f1db911e..4e4f556a39b 100644 --- a/tests/queries/0_stateless/02970_generate_series.reference +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -4,7 +4,6 @@ 8 500000001 50000000 -100000001 0 10 13 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql index 045f584a622..a7f89e1bd3f 100644 --- a/tests/queries/0_stateless/02970_generate_series.sql +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -4,7 +4,6 @@ SELECT count() FROM generate_series(10, 20, 3); SELECT count() FROM generate_series(7, 77, 10); SELECT count() FROM generate_series(0, 1000000000, 2); SELECT count() FROM generate_series(0, 999999999, 20); -SELECT count() FROM generate_series(0, 1000000000, 2) WHERE generate_series % 5 == 0; SELECT * FROM generate_series(5, 4); SELECT * FROM generate_series(0, 0); From 835b47519a7c575d70542e5a37c97dbf5a2b25f9 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 14 Feb 2024 00:44:38 +0100 Subject: [PATCH 023/332] impl --- src/Backups/BackupIO_S3.cpp | 1 + src/Coordination/KeeperSnapshotManagerS3.cpp | 1 + src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 + src/IO/S3/Client.cpp | 19 +++++++++--- src/IO/S3/Client.h | 6 ++++ src/IO/S3/Requests.h | 32 ++++++++++++++++++-- src/IO/S3/URI.cpp | 18 +++++------ src/IO/S3/tests/gtest_aws_s3_client.cpp | 25 ++++++++++++++- src/IO/WriteBufferFromS3.cpp | 15 ++++++++- src/IO/WriteBufferFromS3.h | 1 + src/IO/tests/gtest_s3_uri.cpp | 8 +++++ src/IO/tests/gtest_writebuffer_s3.cpp | 21 +++++++------ src/Storages/StorageS3.cpp | 1 + 13 files changed, 119 insertions(+), 30 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 9359602a651..2063af2061c 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -73,6 +73,7 @@ namespace .use_virtual_addressing = s3_uri.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(s3_uri.endpoint), }; return S3::ClientFactory::instance().create( diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 0337a564660..9779a041095 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -103,6 +103,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo .use_virtual_addressing = new_uri.is_virtual_hosted_style, .disable_checksum = false, .gcs_issue_compose_request = false, + .is_s3express_bucket = S3::isS3ExpressEndpoint(new_uri.endpoint), }; auto client = S3::ClientFactory::instance().create( diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 4fd4b17aabe..b8688cd3de6 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -97,6 +97,7 @@ std::unique_ptr getClient( .use_virtual_addressing = uri.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = config.getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(endpoint), }; return S3::ClientFactory::instance().create( diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 1b6b245b89a..a75d41df3d1 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -304,6 +304,9 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const request.setApiMode(api_mode); + if (isS3ExpressBucket()) + request.setIsS3ExpressBucket(); + addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers); if (auto region = getRegionForBucket(bucket); !region.empty()) @@ -530,7 +533,11 @@ Client::doRequest(RequestType & request, RequestFn request_fn) const addAdditionalAMZHeadersToCanonicalHeadersList(request, client_configuration.extra_headers); const auto & bucket = request.GetBucket(); request.setApiMode(api_mode); - if (client_settings.disable_checksum) + + /// We have to use checksums for S3Express buckets, so the order of checks should be the following + if (client_settings.is_s3express_bucket) + request.setIsS3ExpressBucket(); + else if (client_settings.disable_checksum) request.disableChecksum(); if (auto region = getRegionForBucket(bucket); !region.empty()) @@ -915,9 +922,9 @@ std::unique_ptr ClientFactory::create( // NOLINT std::move(sse_kms_config), credentials_provider, client_configuration, // Client configuration. - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - client_settings - ); + client_settings.is_s3express_bucket ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent + : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + client_settings); } PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT @@ -956,6 +963,10 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT return config; } +bool isS3ExpressEndpoint(const std::string & endpoint) +{ + return endpoint.contains("s3express"); +} } } diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 8da21bd2c2c..c7bc727bf32 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -92,6 +92,8 @@ private: std::unordered_map> client_caches; }; +bool isS3ExpressEndpoint(const std::string & endpoint); + struct ClientSettings { bool use_virtual_addressing; @@ -107,6 +109,7 @@ struct ClientSettings /// Ability to enable it preserved since likely it is required for old /// files. bool gcs_issue_compose_request; + bool is_s3express_bucket; }; /// Client that improves the client from the AWS SDK @@ -208,6 +211,9 @@ public: const std::shared_ptr& httpRequest) const override; bool supportsMultiPartCopy() const; + + bool isS3ExpressBucket() const { return client_settings.is_s3express_bucket; } + private: friend struct ::MockS3::Client; diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index bfb94a5a67e..6f82a0f39d3 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -21,12 +21,32 @@ #include #include #include +#include +#include + +#include namespace DB::S3 { namespace Model = Aws::S3::Model; +/// Used only for S3Express +namespace RequestChecksum +{ +inline void setPartChecksum(Model::CompletedPart & part, const std::string & checksum) +{ + part.SetChecksumCRC32(checksum); +} + +template +inline void setChecksumAlgorithm(R & request) +{ + if constexpr (requires { request.SetChecksumAlgorithm(Model::ChecksumAlgorithm::CRC32); }) + request.SetChecksumAlgorithm(Model::ChecksumAlgorithm::CRC32); +} +}; + template class ExtendedRequest : public BaseRequest { @@ -49,11 +69,13 @@ public: Aws::String GetChecksumAlgorithmName() const override { + chassert(!is_s3express_bucket || checksum); + /// Return empty string is enough to disable checksums (see /// AWSClient::AddChecksumToRequest [1] for more details). /// /// [1]: https://github.com/aws/aws-sdk-cpp/blob/b0ee1c0d336dbb371c34358b68fba6c56aae2c92/src/aws-cpp-sdk-core/source/client/AWSClient.cpp#L783-L839 - if (!checksum) + if (!is_s3express_bucket && !checksum) return ""; return BaseRequest::GetChecksumAlgorithmName(); } @@ -84,9 +106,12 @@ public: } /// Disable checksum to avoid extra read of the input stream - void disableChecksum() const + void disableChecksum() const { checksum = false; } + + void setIsS3ExpressBucket() { - checksum = false; + is_s3express_bucket = true; + RequestChecksum::setChecksumAlgorithm(*this); } protected: @@ -94,6 +119,7 @@ protected: mutable std::optional uri_override; mutable ApiMode api_mode{ApiMode::AWS}; mutable bool checksum = true; + bool is_s3express_bucket = false; }; class CopyObjectRequest : public ExtendedRequest diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 23f59420bfe..062d3b80850 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -35,7 +35,7 @@ URI::URI(const std::string & uri_) /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.Region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); + static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); /// Case when bucket name and key represented in path of S3 URL. /// E.g. (https://s3.Region.amazonaws.com/bucket-name/key) @@ -43,6 +43,7 @@ URI::URI(const std::string & uri_) static const RE2 path_style_pattern("^/([^/]*)/(.*)"); static constexpr auto S3 = "S3"; + static constexpr auto S3EXPRESS = "S3EXPRESS"; static constexpr auto COSN = "COSN"; static constexpr auto COS = "COS"; static constexpr auto OBS = "OBS"; @@ -115,21 +116,16 @@ URI::URI(const std::string & uri_) } boost::to_upper(name); - if (name != S3 && name != COS && name != OBS && name != OSS && name != EOS) + /// For S3Express it will look like s3express-eun1-az1, i.e. contain region and AZ info + if (name != S3 && !name.starts_with(S3EXPRESS) && name != COS && name != OBS && name != OSS && name != EOS) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", quoteString(name)); - if (name == S3) - storage_name = name; - else if (name == OBS) - storage_name = OBS; - else if (name == OSS) - storage_name = OSS; - else if (name == EOS) - storage_name = EOS; - else + if (name == COS || name == COSN) storage_name = COSN; + else + storage_name = name; } else if (re2::RE2::PartialMatch(uri.getPath(), path_style_pattern, &bucket, &key)) { diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index 33917314bca..8edbe12a22f 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -110,7 +110,8 @@ void testServerSideEncryption( bool disable_checksum, String server_side_encryption_customer_key_base64, DB::S3::ServerSideEncryptionKMSConfig sse_kms_config, - String expected_headers) + String expected_headers, + bool is_s3express_bucket = false) { TestPocoHTTPServer http; @@ -144,6 +145,7 @@ void testServerSideEncryption( .use_virtual_addressing = uri.is_virtual_hosted_style, .disable_checksum = disable_checksum, .gcs_issue_compose_request = false, + .is_s3express_bucket = is_s3express_bucket, }; std::shared_ptr client = DB::S3::ClientFactory::instance().create( @@ -295,4 +297,25 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersWrite) "x-amz-server-side-encryption-context: arn:aws:s3:::bucket_ARN\n"); } +TEST(IOTestAwsS3Client, ChecksumHeaderIsPresentForS3Express) +{ + /// See https://github.com/ClickHouse/ClickHouse/pull/19748 + testServerSideEncryption( + doWriteRequest, + /* disable_checksum= */ true, + "", + {}, + "authorization: ... SignedHeaders=" + "amz-sdk-invocation-id;" + "amz-sdk-request;" + "content-length;" + "content-type;" + "host;" + "x-amz-checksum-crc32;" + "x-amz-content-sha256;" + "x-amz-date;" + "x-amz-sdk-checksum-algorithm, ...\n", + /*is_s3express_bucket=*/true); +} + #endif diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 5bb01050591..a162992278f 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -18,7 +18,9 @@ #include #include +#include #include +#include #include @@ -456,6 +458,14 @@ S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, Pa /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 req.SetContentType("binary/octet-stream"); + /// Checksums need to be provided on CompleteMultipartUpload requests, so we calculate then manually and store in multipart_checksums + if (client_ptr->isS3ExpressBucket()) + { + chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); + req.SetChecksumCRC32(Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody())))); + multipart_checksums.push_back(req.GetChecksumCRC32()); + } + return req; } @@ -575,7 +585,10 @@ void WriteBufferFromS3::completeMultipartUpload() for (size_t i = 0; i < multipart_tags.size(); ++i) { Aws::S3::Model::CompletedPart part; - multipart_upload.AddParts(part.WithETag(multipart_tags[i]).WithPartNumber(static_cast(i + 1))); + part.WithETag(multipart_tags[i]).WithPartNumber(static_cast(i + 1)); + if (!multipart_checksums.empty()) + S3::RequestChecksum::setPartChecksum(part, multipart_checksums.at(i)); + multipart_upload.AddParts(part); } req.SetMultipartUpload(multipart_upload); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 5dc269990a1..148cd27f854 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -100,6 +100,7 @@ private: /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. String multipart_upload_id; std::deque multipart_tags; + std::deque multipart_checksums; bool multipart_upload_finished = false; /// Track that prefinalize() is called only once diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index c088e41f1e8..5bf0dfb962d 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -162,6 +162,14 @@ TEST(S3UriTest, validPatterns) ASSERT_EQ("", uri.version_id); ASSERT_EQ(false, uri.is_virtual_hosted_style); } + { + S3::URI uri("https://test-perf-bucket--eun1-az1--x-s3.s3express-eun1-az1.eu-north-1.amazonaws.com/test.csv"); + ASSERT_EQ("https://s3express-eun1-az1.eu-north-1.amazonaws.com", uri.endpoint); + ASSERT_EQ("test-perf-bucket--eun1-az1--x-s3", uri.bucket); + ASSERT_EQ("test.csv", uri.key); + ASSERT_EQ("", uri.version_id); + ASSERT_EQ(true, uri.is_virtual_hosted_style); + } } TEST_P(S3UriTest, invalidPatterns) diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index ae00bb2e9e2..d9cb486c09e 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -205,16 +205,17 @@ struct Client : DB::S3::Client { explicit Client(std::shared_ptr mock_s3_store) : DB::S3::Client( - 100, - DB::S3::ServerSideEncryptionKMSConfig(), - std::make_shared("", ""), - GetClientConfiguration(), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - DB::S3::ClientSettings{ - .use_virtual_addressing = true, - .disable_checksum= false, - .gcs_issue_compose_request = false, - }) + 100, + DB::S3::ServerSideEncryptionKMSConfig(), + std::make_shared("", ""), + GetClientConfiguration(), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + DB::S3::ClientSettings{ + .use_virtual_addressing = true, + .disable_checksum = false, + .gcs_issue_compose_request = false, + .is_s3express_bucket = false, + }) , store(mock_s3_store) {} diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2d8ef3df1c8..044a1ca5362 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1427,6 +1427,7 @@ void StorageS3::Configuration::connect(const ContextPtr & context) .use_virtual_addressing = url.is_virtual_hosted_style, .disable_checksum = local_settings.s3_disable_checksum, .gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false), + .is_s3express_bucket = S3::isS3ExpressEndpoint(url.endpoint), }; auto credentials = Aws::Auth::AWSCredentials(auth_settings.access_key_id, auth_settings.secret_access_key, auth_settings.session_token); From 524a2ca72decc124ef1e38b79843c2388cceb0bb Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 27 Feb 2024 19:17:34 +0100 Subject: [PATCH 024/332] WIP on createForShard --- .../OptimizeShardingKeyRewriteInVisitor.cpp | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 3a592c0fe55..8aca28a90ef 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -5,6 +5,12 @@ #include #include #include +#include "Analyzer/ColumnNode.h" +#include "Analyzer/ConstantNode.h" +#include "Analyzer/FunctionNode.h" +#include "Analyzer/IQueryTreeNode.h" +#include "Analyzer/InDepthQueryTreeVisitor.h" +#include "DataTypes/IDataType.h" namespace { @@ -119,4 +125,42 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d } } + +class OptimizeShardingKeyRewriteIn : InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void enterImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "in") + return; + + auto & arguments = function_node->getArguments().getNodes(); + auto * column = arguments[0]->as(); + if (!column) + return; + + if (!data.sharding_key_expr->getRequiredColumnsWithTypes().contains(column->getColumnName())) + return; + + if (auto * constant = arguments[1]->as()) + { + if (isTuple(constant->getResultType())) + { + auto & tuple = constant->getValue().get(); + std::erase_if(tuple, [&](auto & child) + { + return tuple.size() > 1 && !shardContains(child, name, data); + }); + } + } + } + + OptimizeShardingKeyRewriteInMatcher::Data data; +}; + + } From 0de2d766fa971f54eff40641e16ed6857e1ece5f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 28 Feb 2024 15:30:06 +0100 Subject: [PATCH 025/332] WIP on different JSONs on shards --- src/Analyzer/IdentifierNode.cpp | 10 +- src/Analyzer/IdentifierNode.h | 6 ++ src/DataTypes/ObjectUtils.cpp | 75 ++++++++++++- src/DataTypes/ObjectUtils.h | 11 ++ .../ClusterProxy/SelectStreamFactory.cpp | 79 +++++++++++++- .../ClusterProxy/SelectStreamFactory.h | 26 +++++ .../ClusterProxy/executeQuery.cpp | 101 ++++++++++++------ src/Interpreters/ClusterProxy/executeQuery.h | 2 - .../OptimizeShardingKeyRewriteInVisitor.cpp | 35 ++++-- .../OptimizeShardingKeyRewriteInVisitor.h | 3 + src/Processors/QueryPlan/ReadFromRemote.cpp | 10 +- src/Storages/StorageDistributed.cpp | 32 +++--- 12 files changed, 325 insertions(+), 65 deletions(-) diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index 88b3daacb12..7e4d4c02a4c 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -56,13 +57,18 @@ void IdentifierNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr IdentifierNode::cloneImpl() const { - return std::make_shared(identifier); + auto result = std::make_shared(identifier); + result->use_parts_for_to_ast = use_parts_for_to_ast; + return result; } ASTPtr IdentifierNode::toASTImpl(const ConvertToASTOptions & /* options */) const { auto identifier_parts = identifier.getParts(); - return std::make_shared(std::move(identifier_parts)); + if (use_parts_for_to_ast) + return std::make_shared(std::move(identifier_parts)); + else + return std::make_shared(identifier.getFullName()); } } diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 872bb14d512..3bc37b4c69d 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -52,6 +52,11 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; + void useFullNameInToAST() + { + use_parts_for_to_ast = false; + } + protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; @@ -64,6 +69,7 @@ protected: private: Identifier identifier; std::optional table_expression_modifiers; + bool use_parts_for_to_ast = false; static constexpr size_t children_size = 0; }; diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 23d29136c85..01ba50d90f3 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -20,6 +21,16 @@ #include #include #include +#include "Analyzer/ConstantNode.h" +#include "Analyzer/FunctionNode.h" +#include "Analyzer/IQueryTreeNode.h" +#include "Analyzer/Identifier.h" +#include "Analyzer/IdentifierNode.h" +#include "Analyzer/QueryNode.h" +#include "Analyzer/Utils.h" +#include +#include +#include "Common/logger_useful.h" namespace DB @@ -888,10 +899,10 @@ static void addConstantToWithClause(const ASTPtr & query, const String & column_ /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -void replaceMissedSubcolumnsByConstants( +NamesAndTypes calculateMissedSubcolumns( const ColumnsDescription & expected_columns, - const ColumnsDescription & available_columns, - ASTPtr query) + const ColumnsDescription & available_columns +) { NamesAndTypes missed_names_types; @@ -928,6 +939,18 @@ void replaceMissedSubcolumnsByConstants( [](const auto & lhs, const auto & rhs) { return lhs.name < rhs.name; }); } + return missed_names_types; +} + +/// @expected_columns and @available_columns contain descriptions +/// of extended Object columns. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + ASTPtr query) +{ + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); + if (missed_names_types.empty()) return; @@ -940,6 +963,52 @@ void replaceMissedSubcolumnsByConstants( addConstantToWithClause(query, name, type); } +/// @expected_columns and @available_columns contain descriptions +/// of extended Object columns. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + QueryTreeNodePtr & query, + const ContextPtr & context [[maybe_unused]]) +{ + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); + + if (missed_names_types.empty()) + return; + + auto * query_node = query->as(); + if (!query_node) + return; + + auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); + + auto & with_nodes = query_node->getWith().getNodes(); + + std::unordered_map column_name_to_node; + for (const auto & [name, type] : missed_names_types) + { + auto constant = std::make_shared(type->getDefault(), type); + constant->setAlias(table_expression->getAlias() + name); + // auto materialize = std::make_shared("materialize"); + + // auto function = FunctionFactory::instance().get("materialize", context); + // materialize->getArguments().getNodes() = { constant }; + // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); + // materialize->setAlias(name); + + with_nodes.push_back(constant); + + auto id = std::make_shared(Identifier(table_expression->getAlias() + name)); + id->useFullNameInToAST(); + column_name_to_node[name] = id; + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); + } + + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); + replaceColumns(query, table_expression, column_name_to_node); + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); +} + Field FieldVisitorReplaceScalars::operator()(const Array & x) const { if (num_dimensions_to_keep == 0) diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2bfcaae09ca..f4a8abe8abf 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -3,6 +3,8 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" +#include "Interpreters/Context_fwd.h" #include #include #include @@ -14,6 +16,9 @@ namespace DB struct StorageSnapshot; using StorageSnapshotPtr = std::shared_ptr; +class IQueryTreeNode; +using QueryTreeNodePtr = std::shared_ptr; + /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); @@ -97,6 +102,12 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + QueryTreeNodePtr & query, + const ContextPtr & context); + /// Visitor that keeps @num_dimensions_to_keep dimensions in arrays /// and replaces all scalars or nested arrays to @replacement at that level. class FieldVisitorReplaceScalars : public StaticVisitor diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index f0592735caf..5167ffc0e27 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,6 +5,10 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" +#include "Interpreters/InterpreterSelectQueryAnalyzer.h" +#include "Interpreters/SelectQueryOptions.h" +#include "Planner/Utils.h" #include #include #include @@ -124,18 +128,55 @@ void SelectStreamFactory::createForShard( if (it != objects_by_shard.end()) replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast); + createForShardImpl( + shard_info, + query_ast, + main_table, + table_func_ptr, + std::move(context), + local_plans, + remote_shards, + shard_count, + parallel_replicas_enabled, + std::move(shard_filter_generator)); +} + +void SelectStreamFactory::createForShardImpl( + const Cluster::ShardInfo & shard_info, + const ASTPtr & query_ast, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator) +{ auto emplace_local_stream = [&]() { + Block shard_header; + if (context->getSettingsRef().allow_experimental_analyzer) + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + else + shard_header = header; + local_plans.emplace_back(createLocalPlan( - query_ast, header, context, processed_stage, shard_info.shard_num, shard_count)); + query_ast, shard_header, context, processed_stage, shard_info.shard_num, shard_count)); }; auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0) { + Block shard_header; + if (context->getSettingsRef().allow_experimental_analyzer) + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + else + shard_header = header; + remote_shards.emplace_back(Shard{ .query = query_ast, .main_table = main_table, - .header = header, + .header = shard_header, .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -243,6 +284,40 @@ void SelectStreamFactory::createForShard( emplace_remote_stream(); } +void SelectStreamFactory::createForShard( + const Cluster::ShardInfo & shard_info, + const QueryTreeNodePtr & query_tree, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator) +{ + + auto it = objects_by_shard.find(shard_info.shard_num); + QueryTreeNodePtr modified_query = query_tree; + if (it != objects_by_shard.end()) + replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + + auto query_ast = queryNodeToDistributedSelectQuery(modified_query); + + createForShardImpl( + shard_info, + query_ast, + main_table, + table_func_ptr, + std::move(context), + local_plans, + remote_shards, + shard_count, + parallel_replicas_enabled, + std::move(shard_filter_generator)); + +} + } } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 9993ea7028d..45d6ea14c01 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -7,6 +7,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" namespace DB { @@ -83,10 +84,35 @@ public: bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator); + void createForShard( + const Cluster::ShardInfo & shard_info, + const QueryTreeNodePtr & query_tree, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator); + const Block header; const ColumnsDescriptionByShardNum objects_by_shard; const StorageSnapshotPtr storage_snapshot; QueryProcessingStage::Enum processed_stage; + +private: + void createForShardImpl( + const Cluster::ShardInfo & shard_info, + const ASTPtr & query_ast, + const StorageID & main_table, + const ASTPtr & table_func_ptr, + ContextPtr context, + std::vector & local_plans, + Shards & remote_shards, + UInt32 shard_count, + bool parallel_replicas_enabled, + AdditionalShardFilterGenerator shard_filter_generator); }; } diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 6cdff939af1..07ef7aa6c96 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -204,12 +204,10 @@ void executeQuery( const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, LoggerPtr log, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster, const DistributedSettings & distributed_settings, AdditionalShardFilterGenerator shard_filter_generator) { @@ -218,6 +216,8 @@ void executeQuery( if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); + const ClusterPtr & not_optimized_cluster = query_info.cluster; + std::vector plans; SelectStreamFactory::Shards remote_shards; @@ -237,40 +237,81 @@ void executeQuery( new_context->increaseDistributedDepth(); const size_t shards = cluster->getShardCount(); - for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) + + if (context->getSettingsRef().allow_experimental_analyzer) { - const auto & shard_info = cluster->getShardsInfo()[i]; - - ASTPtr query_ast_for_shard = query_ast->clone(); - if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) { - OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ - sharding_key_expr, - sharding_key_expr->getSampleBlock().getByPosition(0).type, - sharding_key_column_name, + const auto & shard_info = cluster->getShardsInfo()[i]; + + auto query_for_shard = query_info.query_tree->clone(); + if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + { + OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ + sharding_key_expr, + sharding_key_expr->getSampleBlock().getByPosition(0).type, + sharding_key_column_name, + shard_info, + not_optimized_cluster->getSlotToShard(), + }; + optimizeShardingKeyRewriteIn(query_for_shard, std::move(visitor_data), new_context); + } + + // decide for each shard if parallel reading from replicas should be enabled + // according to settings and number of replicas declared per shard + const auto & addresses = cluster->getShardsAddresses().at(i); + bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + + stream_factory.createForShard( shard_info, - not_optimized_cluster->getSlotToShard(), - }; - OptimizeShardingKeyRewriteInVisitor visitor(visitor_data); - visitor.visit(query_ast_for_shard); + query_for_shard, + main_table, + table_func_ptr, + new_context, + plans, + remote_shards, + static_cast(shards), + parallel_replicas_enabled, + shard_filter_generator); } + } + else + { + for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) + { + const auto & shard_info = cluster->getShardsInfo()[i]; - // decide for each shard if parallel reading from replicas should be enabled - // according to settings and number of replicas declared per shard - const auto & addresses = cluster->getShardsAddresses().at(i); - bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + ASTPtr query_ast_for_shard = query_info.query->clone(); + if (sharding_key_expr && query_info.optimized_cluster && settings.optimize_skip_unused_shards_rewrite_in && shards > 1) + { + OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ + sharding_key_expr, + sharding_key_expr->getSampleBlock().getByPosition(0).type, + sharding_key_column_name, + shard_info, + not_optimized_cluster->getSlotToShard(), + }; + OptimizeShardingKeyRewriteInVisitor visitor(visitor_data); + visitor.visit(query_ast_for_shard); + } - stream_factory.createForShard( - shard_info, - query_ast_for_shard, - main_table, - table_func_ptr, - new_context, - plans, - remote_shards, - static_cast(shards), - parallel_replicas_enabled, - shard_filter_generator); + // decide for each shard if parallel reading from replicas should be enabled + // according to settings and number of replicas declared per shard + const auto & addresses = cluster->getShardsAddresses().at(i); + bool parallel_replicas_enabled = addresses.size() > 1 && context->canUseTaskBasedParallelReplicas(); + + stream_factory.createForShard( + shard_info, + query_ast_for_shard, + main_table, + table_func_ptr, + new_context, + plans, + remote_shards, + static_cast(shards), + parallel_replicas_enabled, + shard_filter_generator); + } } if (!remote_shards.empty()) diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index bbc3c6c9e49..8f6f6300c7b 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -58,12 +58,10 @@ void executeQuery( const ASTPtr & table_func_ptr, SelectStreamFactory & stream_factory, LoggerPtr log, - const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster, const DistributedSettings & distributed_settings, AdditionalShardFilterGenerator shard_filter_generator); diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 8aca28a90ef..42c6e63da01 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -11,6 +12,7 @@ #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/InDepthQueryTreeVisitor.h" #include "DataTypes/IDataType.h" +#include "Interpreters/Context_fwd.h" namespace { @@ -126,11 +128,15 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d } -class OptimizeShardingKeyRewriteIn : InDepthQueryTreeVisitorWithContext +class OptimizeShardingKeyRewriteIn : public InDepthQueryTreeVisitorWithContext { public: using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; + + OptimizeShardingKeyRewriteIn(OptimizeShardingKeyRewriteInVisitor::Data data_, ContextPtr context) + : Base(std::move(context)) + , data(std::move(data_)) + {} void enterImpl(QueryTreeNodePtr & node) { @@ -143,6 +149,8 @@ public: if (!column) return; + auto name = column->getColumnName(); + if (!data.sharding_key_expr->getRequiredColumnsWithTypes().contains(column->getColumnName())) return; @@ -150,17 +158,30 @@ public: { if (isTuple(constant->getResultType())) { - auto & tuple = constant->getValue().get(); - std::erase_if(tuple, [&](auto & child) + const auto & tuple = constant->getValue().get(); + Tuple new_tuple; + new_tuple.reserve(tuple.size()); + + for (const auto & child : tuple) { - return tuple.size() > 1 && !shardContains(child, name, data); - }); + if (shardContains(child, name, data)) + new_tuple.push_back(child); + } + + if (new_tuple.empty()) + new_tuple.push_back(tuple.back()); + node = std::make_shared(new_tuple); } } } - OptimizeShardingKeyRewriteInMatcher::Data data; + OptimizeShardingKeyRewriteInVisitor::Data data; }; +void optimizeShardingKeyRewriteIn(QueryTreeNodePtr & node, OptimizeShardingKeyRewriteInVisitor::Data data, ContextPtr context) +{ + OptimizeShardingKeyRewriteIn visitor(std::move(data), std::move(context)); + visitor.visit(node); +} } diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h index d546db40df7..d202609160b 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h @@ -2,6 +2,7 @@ #include #include +#include "Analyzer/IQueryTreeNode.h" namespace DB { @@ -44,4 +45,6 @@ struct OptimizeShardingKeyRewriteInMatcher using OptimizeShardingKeyRewriteInVisitor = InDepthNodeVisitor; +void optimizeShardingKeyRewriteIn(QueryTreeNodePtr & node, OptimizeShardingKeyRewriteInVisitor::Data data, ContextPtr context); + } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 93c73a66b78..022c4f699f2 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -216,7 +216,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -281,7 +281,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact auto remote_query_executor = std::make_shared( shard.shard_info.pool, query_string, - output_stream->header, + shard.header, context, throttler, scalars, @@ -297,7 +297,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } } else @@ -305,7 +305,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact const String query_string = formattedAST(shard.query); auto remote_query_executor = std::make_shared( - shard.shard_info.pool, query_string, output_stream->header, context, throttler, scalars, external_tables, stage); + shard.shard_info.pool, query_string, shard.header, context, throttler, scalars, external_tables, stage); remote_query_executor->setLogger(log); if (context->canUseTaskBasedParallelReplicas()) @@ -326,7 +326,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), shard.header); } } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 92e7dcdf4f2..34ab21a4751 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,6 +30,7 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" #include #include @@ -813,7 +814,8 @@ void StorageDistributed::read( const size_t /*num_streams*/) { Block header; - ASTPtr query_ast; + + SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) { @@ -821,7 +823,7 @@ void StorageDistributed::read( if (!remote_table_function_ptr) remote_storage_id = StorageID{remote_database, remote_table}; - auto query_tree_distributed = buildQueryTreeDistributed(query_info, + auto query_tree_distributed = buildQueryTreeDistributed(modified_query_info, storage_snapshot, remote_storage_id, remote_table_function_ptr); @@ -831,20 +833,24 @@ void StorageDistributed::read( */ for (auto & column : header) column.column = column.column->convertToFullColumnIfConst(); - query_ast = queryNodeToDistributedSelectQuery(query_tree_distributed); + modified_query_info.query = queryNodeToDistributedSelectQuery(query_tree_distributed); + + modified_query_info.query_tree = std::move(query_tree_distributed); } else { - header = InterpreterSelectQuery(query_info.query, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); - query_ast = query_info.query; + header = InterpreterSelectQuery(modified_query_info.query, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); } - const auto & modified_query_ast = ClusterProxy::rewriteSelectQuery( - local_context, query_ast, - remote_database, remote_table, remote_table_function_ptr); + if (!local_context->getSettingsRef().allow_experimental_analyzer) + { + modified_query_info.query = ClusterProxy::rewriteSelectQuery( + local_context, modified_query_info.query, + remote_database, remote_table, remote_table_function_ptr); + } /// Return directly (with correct header) if no shard to query. - if (query_info.getCluster()->getShardsInfo().empty()) + if (modified_query_info.getCluster()->getShardsInfo().empty()) { if (local_context->getSettingsRef().allow_experimental_analyzer) return; @@ -872,7 +878,7 @@ void StorageDistributed::read( const auto & settings = local_context->getSettingsRef(); ClusterProxy::AdditionalShardFilterGenerator additional_shard_filter_generator; - if (local_context->canUseParallelReplicasCustomKey(*query_info.getCluster())) + if (local_context->canUseParallelReplicasCustomKey(*modified_query_info.getCluster())) { if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *local_context)) { @@ -881,7 +887,7 @@ void StorageDistributed::read( column_description = this->getInMemoryMetadataPtr()->columns, custom_key_type = settings.parallel_replicas_custom_key_filter_type.value, context = local_context, - replica_count = query_info.getCluster()->getShardsInfo().front().per_replica_pools.size()](uint64_t replica_num) -> ASTPtr + replica_count = modified_query_info.getCluster()->getShardsInfo().front().per_replica_pools.size()](uint64_t replica_num) -> ASTPtr { return getCustomKeyFilterForParallelReplica( replica_count, replica_num - 1, my_custom_key_ast, custom_key_type, column_description, context); @@ -897,12 +903,10 @@ void StorageDistributed::read( remote_table_function_ptr, select_stream_factory, log, - modified_query_ast, local_context, - query_info, + modified_query_info, sharding_key_expr, sharding_key_column_name, - query_info.cluster, distributed_settings, additional_shard_filter_generator); From 8bf7c2c5971afc22dda32f9f4ad453ac481f2359 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 28 Feb 2024 15:40:42 +0100 Subject: [PATCH 026/332] Use output header --- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 022c4f699f2..fde2313bc15 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -216,7 +216,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -297,7 +297,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } } else @@ -326,7 +326,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), shard.header); + addConvertingActions(pipes.back(), output_stream->header); } } From 974ba7364f193838f735a9233c6dec4298172542 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 29 Feb 2024 00:55:17 +0100 Subject: [PATCH 027/332] better --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 16 +++++++++++++--- src/IO/S3/Client.cpp | 2 ++ src/IO/S3/URI.cpp | 2 +- src/IO/WriteBufferFromS3.h | 2 +- src/Storages/StorageS3.cpp | 4 ++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index b8688cd3de6..10172805f06 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -1,5 +1,6 @@ #include -#include "IO/S3/Client.h" +#include +#include #if USE_AWS_S3 @@ -10,7 +11,7 @@ #include #include #include -#include "Disks/DiskFactory.h" +#include #include #include @@ -25,6 +26,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int NO_ELEMENTS_IN_CONFIG; +} + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { const Settings & settings = context->getSettingsRef(); @@ -47,11 +53,15 @@ std::unique_ptr getClient( const Settings & global_settings = context->getGlobalContext()->getSettingsRef(); const Settings & local_settings = context->getSettingsRef(); - String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); + const String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); if (!uri.key.ends_with('/')) uri.key.push_back('/'); + if (S3::isS3ExpressEndpoint(endpoint) && !config.has(config_prefix + ".region")) + throw Exception( + ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Region should be explicitly specified for directory buckets ({})", config_prefix); + S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( config.getString(config_prefix + ".region", ""), context->getRemoteHostFilter(), diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index a75d41df3d1..4f93aba2f84 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -1,4 +1,5 @@ #include +#include #if USE_AWS_S3 @@ -965,6 +966,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT bool isS3ExpressEndpoint(const std::string & endpoint) { + /// On one hand this check isn't 100% reliable, on the other - all it will change is whether we attach checksums to the requests. return endpoint.contains("s3express"); } } diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 062d3b80850..027cb624ed5 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -122,7 +122,7 @@ URI::URI(const std::string & uri_) "Object storage system name is unrecognized in virtual hosted style S3 URI: {}", quoteString(name)); - if (name == COS || name == COSN) + if (name == COS) storage_name = COSN; else storage_name = name; diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 148cd27f854..59f4e19e15b 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -100,7 +100,7 @@ private: /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. String multipart_upload_id; std::deque multipart_tags; - std::deque multipart_checksums; + std::deque multipart_checksums; // if enabled bool multipart_upload_finished = false; /// Track that prefinalize() is called only once diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 044a1ca5362..f96ff8b7eb6 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -133,6 +133,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int CANNOT_COMPILE_REGEXP; extern const int FILE_DOESNT_EXIST; + extern const int NO_ELEMENTS_IN_CONFIG; } @@ -1403,6 +1404,9 @@ void StorageS3::Configuration::connect(const ContextPtr & context) const Settings & global_settings = context->getGlobalContext()->getSettingsRef(); const Settings & local_settings = context->getSettingsRef(); + if (S3::isS3ExpressEndpoint(url.endpoint) && auth_settings.region.empty()) + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Region should be explicitly specified for directory buckets"); + S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( auth_settings.region, context->getRemoteHostFilter(), From 37917a3ed34df22756562a04a90d3c985ca23bd8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 29 Feb 2024 01:42:32 +0100 Subject: [PATCH 028/332] better --- src/IO/S3/Requests.h | 12 ++++++++++++ src/IO/WriteBufferFromS3.cpp | 10 +++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 6f82a0f39d3..196f074c9df 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -39,6 +40,17 @@ inline void setPartChecksum(Model::CompletedPart & part, const std::string & che part.SetChecksumCRC32(checksum); } +inline void setRequestChecksum(Model::UploadPartRequest & req, const std::string & checksum) +{ + req.SetChecksumCRC32(checksum); +} + +inline std::string calculateChecksum(Model::UploadPartRequest & req) +{ + chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody()))); +} + template inline void setChecksumAlgorithm(R & request) { diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index a162992278f..80ca96b0382 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -18,10 +18,6 @@ #include #include -#include -#include -#include - #include @@ -461,9 +457,9 @@ S3::UploadPartRequest WriteBufferFromS3::getUploadRequest(size_t part_number, Pa /// Checksums need to be provided on CompleteMultipartUpload requests, so we calculate then manually and store in multipart_checksums if (client_ptr->isS3ExpressBucket()) { - chassert(req.GetChecksumAlgorithm() == Aws::S3::Model::ChecksumAlgorithm::CRC32); - req.SetChecksumCRC32(Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(*(req.GetBody())))); - multipart_checksums.push_back(req.GetChecksumCRC32()); + auto checksum = S3::RequestChecksum::calculateChecksum(req); + S3::RequestChecksum::setRequestChecksum(req, checksum); + multipart_checksums.push_back(std::move(checksum)); } return req; From 8b1a1d42daa01e946aa8102d683dbab90b447838 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Thu, 29 Feb 2024 18:07:00 +0300 Subject: [PATCH 029/332] Traverse shadow directory for system.remote_data_paths --- src/Core/Settings.h | 1 + src/Disks/IDisk.h | 5 ++++- src/Disks/ObjectStorages/DiskObjectStorage.cpp | 10 ++++++++-- src/Disks/ObjectStorages/DiskObjectStorage.h | 5 ++++- .../System/StorageSystemRemoteDataPaths.cpp | 13 +++++++++++++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d77b3a45188..7cf068d7f1f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -843,6 +843,7 @@ class IColumn; M(Bool, optimize_uniq_to_count, true, "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.", 0) \ M(Bool, use_variant_as_common_type, false, "Use Variant as a result type for if/multiIf in case when there is no common type for arguments", 0) \ M(Bool, enable_order_by_all, true, "Enable sorting expression ORDER BY ALL.", 0) \ + M(Bool, traverse_shadow_remote_data_paths, false, "Traverse shadow directory when query system.remote_data_paths", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_materialized_postgresql_table, false, "Allows to use the MaterializedPostgreSQL table engine. Disabled by default, because this feature is experimental", 0) \ diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 3d228850537..62b02938d1a 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -320,7 +320,10 @@ public: {} }; - virtual void getRemotePathsRecursive(const String &, std::vector &) + virtual void getRemotePathsRecursive( + const String &, + std::vector &, + const std::function & /* skip_predicate */ = {}) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 2a648f28f14..460d242d5cd 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -90,11 +90,17 @@ StoredObjects DiskObjectStorage::getStorageObjects(const String & local_path) co return metadata_storage->getStorageObjects(local_path); } -void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) +void DiskObjectStorage::getRemotePathsRecursive( + const String & local_path, + std::vector & paths_map, + const std::function & skip_predicate) { if (!metadata_storage->exists(local_path)) return; + if (skip_predicate && skip_predicate(local_path)) + return; + /// Protect against concurrent delition of files (for example because of a merge). if (metadata_storage->isFile(local_path)) { @@ -142,7 +148,7 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: } for (; it->isValid(); it->next()) - DiskObjectStorage::getRemotePathsRecursive(fs::path(local_path) / it->name(), paths_map); + DiskObjectStorage::getRemotePathsRecursive(fs::path(local_path) / it->name(), paths_map, skip_predicate); } } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index e1576509713..d7af656bea3 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -48,7 +48,10 @@ public: StoredObjects getStorageObjects(const String & local_path) const override; - void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; + void getRemotePathsRecursive( + const String & local_path, + std::vector & paths_map, + const std::function & skip_predicate = {}) override; const std::string & getCacheName() const override { return object_storage->getCacheName(); } diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 87b7a84e8ba..708c1369965 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -9,6 +9,7 @@ #include #include +namespace fs = std::filesystem; namespace DB { @@ -59,6 +60,18 @@ Pipe StorageSystemRemoteDataPaths::read( std::vector remote_paths_by_local_path; disk->getRemotePathsRecursive("store", remote_paths_by_local_path); disk->getRemotePathsRecursive("data", remote_paths_by_local_path); + if (context->getSettingsRef().traverse_shadow_remote_data_paths) + disk->getRemotePathsRecursive( + "shadow", + remote_paths_by_local_path, + [](const String & local_path) + { + // `shadow/{backup_name}/revision.txt` is not an object metadata file + const auto path = fs::path(local_path); + return path.filename() == "revision.txt" && + path.parent_path().has_parent_path() && + path.parent_path().parent_path().filename() == "shadow"; + }); FileCachePtr cache; From 55053dae4459b1d1a6c05d436d1ab421a96c3934 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 29 Feb 2024 19:18:06 +0100 Subject: [PATCH 030/332] Some progress --- src/Analyzer/IdentifierNode.cpp | 10 +--- src/Analyzer/IdentifierNode.h | 6 --- src/DataTypes/ObjectUtils.cpp | 24 +++++----- src/DataTypes/ObjectUtils.h | 4 +- .../ClusterProxy/SelectStreamFactory.cpp | 10 ++-- .../ClusterProxy/SelectStreamFactory.h | 8 +++- src/Processors/QueryPlan/ReadFromRemote.cpp | 47 +++++++++++++++++++ .../test_distributed_type_object/test.py | 2 +- 8 files changed, 80 insertions(+), 31 deletions(-) diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index 7e4d4c02a4c..88b3daacb12 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -57,18 +56,13 @@ void IdentifierNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr IdentifierNode::cloneImpl() const { - auto result = std::make_shared(identifier); - result->use_parts_for_to_ast = use_parts_for_to_ast; - return result; + return std::make_shared(identifier); } ASTPtr IdentifierNode::toASTImpl(const ConvertToASTOptions & /* options */) const { auto identifier_parts = identifier.getParts(); - if (use_parts_for_to_ast) - return std::make_shared(std::move(identifier_parts)); - else - return std::make_shared(identifier.getFullName()); + return std::make_shared(std::move(identifier_parts)); } } diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 3bc37b4c69d..872bb14d512 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -52,11 +52,6 @@ public: void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; - void useFullNameInToAST() - { - use_parts_for_to_ast = false; - } - protected: bool isEqualImpl(const IQueryTreeNode & rhs) const override; @@ -69,7 +64,6 @@ protected: private: Identifier identifier; std::optional table_expression_modifiers; - bool use_parts_for_to_ast = false; static constexpr size_t children_size = 0; }; diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 01ba50d90f3..47d8c5c9113 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -965,30 +965,32 @@ void replaceMissedSubcolumnsByConstants( /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -void replaceMissedSubcolumnsByConstants( +MissingObjectList replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, const ContextPtr & context [[maybe_unused]]) { + MissingObjectList missed_list; + NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); if (missed_names_types.empty()) - return; + return missed_list; auto * query_node = query->as(); if (!query_node) - return; + return missed_list; + + missed_list.reserve(missed_names_types.size()); auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); - auto & with_nodes = query_node->getWith().getNodes(); - std::unordered_map column_name_to_node; for (const auto & [name, type] : missed_names_types) { auto constant = std::make_shared(type->getDefault(), type); - constant->setAlias(table_expression->getAlias() + name); + constant->setAlias(table_expression->getAlias() + "." + name); // auto materialize = std::make_shared("materialize"); // auto function = FunctionFactory::instance().get("materialize", context); @@ -996,17 +998,17 @@ void replaceMissedSubcolumnsByConstants( // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); // materialize->setAlias(name); - with_nodes.push_back(constant); - - auto id = std::make_shared(Identifier(table_expression->getAlias() + name)); - id->useFullNameInToAST(); - column_name_to_node[name] = id; + column_name_to_node[name] = buildCastFunction(constant, type, context); + missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); + LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "{} -> {}", missed_list.back().first, missed_list.back().second); LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); } LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); replaceColumns(query, table_expression, column_name_to_node); LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); + + return missed_list; } Field FieldVisitorReplaceScalars::operator()(const Array & x) const diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index f4a8abe8abf..013e525832e 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -102,7 +102,9 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); -void replaceMissedSubcolumnsByConstants( +using MissingObjectList = std::vector>; + +MissingObjectList replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5167ffc0e27..5bcd1ce68cb 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -151,7 +151,8 @@ void SelectStreamFactory::createForShardImpl( Shards & remote_shards, UInt32 shard_count, bool parallel_replicas_enabled, - AdditionalShardFilterGenerator shard_filter_generator) + AdditionalShardFilterGenerator shard_filter_generator, + MissingObjectList missed_list) { auto emplace_local_stream = [&]() { @@ -177,6 +178,7 @@ void SelectStreamFactory::createForShardImpl( .query = query_ast, .main_table = main_table, .header = shard_header, + .missing_object_list = std::move(missed_list), .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -299,8 +301,9 @@ void SelectStreamFactory::createForShard( auto it = objects_by_shard.find(shard_info.shard_num); QueryTreeNodePtr modified_query = query_tree; + MissingObjectList missed_list; if (it != objects_by_shard.end()) - replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + missed_list = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); auto query_ast = queryNodeToDistributedSelectQuery(modified_query); @@ -314,7 +317,8 @@ void SelectStreamFactory::createForShard( remote_shards, shard_count, parallel_replicas_enabled, - std::move(shard_filter_generator)); + std::move(shard_filter_generator), + std::move(missed_list)); } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 45d6ea14c01..bee7edb3c19 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -43,6 +44,8 @@ ASTPtr rewriteSelectQuery( using ColumnsDescriptionByShardNum = std::unordered_map; using AdditionalShardFilterGenerator = std::function; +using MissingObjectList = std::vector>; + class SelectStreamFactory { public: @@ -55,6 +58,8 @@ public: StorageID main_table; Block header; + MissingObjectList missing_object_list; + Cluster::ShardInfo shard_info; /// If we connect to replicas lazily. @@ -112,7 +117,8 @@ private: Shards & remote_shards, UInt32 shard_count, bool parallel_replicas_enabled, - AdditionalShardFilterGenerator shard_filter_generator); + AdditionalShardFilterGenerator shard_filter_generator, + MissingObjectList missed_list = {}); }; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index fde2313bc15..ac507c6d555 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include "DataTypes/ObjectUtils.h" #include #include #include @@ -31,6 +33,48 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static void addRenamingActions(Pipe & pipe, const MissingObjectList & missed_list, const Block & output_header) +{ + if (missed_list.empty()) + return; + + const auto & output_columns = output_header.getColumnsWithTypeAndName(); + std::vector indexes; + for (size_t i = 0; i < output_columns.size(); ++i) + { + bool found = false; + for (auto const & elem : missed_list) + { + if (output_columns[i].name.contains(elem.second)) + { + found = true; + break; + } + } + if (found) + indexes.push_back(i); + } + + auto dag = std::make_shared(pipe.getHeader().getColumnsWithTypeAndName()); + + for (size_t index : indexes) + { + dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs()[index], output_header.getByPosition(index).name)); + } + + // dag->addAliases(rename_to_apply); + + auto convert_actions = std::make_shared(dag); + pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr + { + return std::make_shared(cur_header, convert_actions); + }); + + LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "EXPECTED:\n{}", output_header.dumpStructure()); + + LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "{}", pipe.getHeader().dumpStructure()); +} + static void addConvertingActions(Pipe & pipe, const Block & header) { if (blocksHaveEqualStructure(pipe.getHeader(), header)) @@ -216,6 +260,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } @@ -297,6 +342,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } } @@ -326,6 +372,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); + addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); addConvertingActions(pipes.back(), output_stream->header); } } diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index b2179af8a3f..f77e0248f02 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -59,7 +59,7 @@ def test_distributed_type_object(started_cluster): ) expected = TSV("120\n") - assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table")) == expected + assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0")) == expected node1.query("TRUNCATE TABLE local_table") node2.query("TRUNCATE TABLE local_table") From 41deadda359ca02528fa6ffe9ecfed09c36b364a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Feb 2024 18:37:00 +0000 Subject: [PATCH 031/332] Automatic style fix --- tests/integration/test_distributed_type_object/test.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index f77e0248f02..7e6c000cb8e 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -59,7 +59,14 @@ def test_distributed_type_object(started_cluster): ) expected = TSV("120\n") - assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0")) == expected + assert ( + TSV( + node1.query( + "SELECT sum(data.k2 * id) FROM dist_table SETTINGS optimize_arithmetic_operations_in_aggregate_functions = 0" + ) + ) + == expected + ) node1.query("TRUNCATE TABLE local_table") node2.query("TRUNCATE TABLE local_table") From a6cebad52bf4f29984db99cd4d4aa1eb41c50895 Mon Sep 17 00:00:00 2001 From: Nataly Merezhuk Date: Thu, 29 Feb 2024 16:32:29 -0500 Subject: [PATCH 032/332] Adds note on supported PostgreSQL versions. --- docs/en/engines/table-engines/integrations/postgresql.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 131df1a435b..9cc4b11243e 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -8,6 +8,10 @@ sidebar_label: PostgreSQL The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data that is stored on a remote PostgreSQL server. +:::note +Currently, only PostgreSQL versions 12 and up are supported. +::: + ## Creating a Table {#creating-a-table} ``` sql From 6143986b6d79c0262f5f7dc3052ec2a3f4cfc490 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Fri, 1 Mar 2024 14:55:02 +0300 Subject: [PATCH 033/332] Add query test --- ...raverse_shadow_system_data_paths.reference | 3 ++ ...03000_traverse_shadow_system_data_paths.sh | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference create mode 100755 tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh new file mode 100755 index 00000000000..a22cb200f9a --- /dev/null +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE="03000_traverse_shadow_system_data_path_table" +BACKUP="03000_traverse_shadow_system_data_path_backup" + +${CLICKHOUSE_CLIENT} --query="CREATE TABLE ${TABLE} ( + id Int64, + data String +) ENGINE=MergeTree() +ORDER BY id +SETTINGS storage_policy='s3_cache';" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO ${TABLE} VALUES (0, 'data');" +${CLICKHOUSE_CLIENT} --query "SELECT count() > 0 FROM system.remote_data_paths WHERE disk_name = 's3_cache'" + +${CLICKHOUSE_CLIENT} --query="ALTER TABLE ${TABLE} FREEZE WITH NAME '${BACKUP}';" +${CLICKHOUSE_CLIENT} --query="DROP TABLE ${TABLE} SYNC;" + +${CLICKHOUSE_CLIENT} --query " + SELECT count() > 0 + FROM system.remote_data_paths + WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' + SETTINGS traverse_shadow_remote_data_paths=1;" +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" >/dev/null +${CLICKHOUSE_CLIENT} --query " + SELECT count() == 0 + FROM system.remote_data_paths + WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' + SETTINGS traverse_shadow_remote_data_paths=1;" From 81185815a48b36d344bda623dd175c30e9b87ba3 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Mon, 4 Mar 2024 14:09:31 +0300 Subject: [PATCH 034/332] Update settings_changes_history --- src/Core/SettingsChangesHistory.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 4805df46d9b..b8793f437d8 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,6 +85,9 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"24.3", { + {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, + }}, {"24.2", { {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, From fbdc5e305365e9d93b86ed47144ffb13c1ce70c1 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Mon, 4 Mar 2024 17:16:51 +0300 Subject: [PATCH 035/332] Ignore flaky fail of system unfreeze --- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index a22cb200f9a..2905d7801ca 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,7 +26,7 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" >/dev/null +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true ${CLICKHOUSE_CLIENT} --query " SELECT count() == 0 FROM system.remote_data_paths From 758a75c1b46fa27a88e3dcf6e70a18dcf41d62ef Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 09:53:30 +0300 Subject: [PATCH 036/332] Fix flaky test. Fix clang-tidy warning --- src/Disks/IDisk.h | 7 +++---- src/Disks/ObjectStorages/DiskObjectStorage.h | 2 +- src/Storages/System/StorageSystemRemoteDataPaths.cpp | 4 ++-- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 7 +------ 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 62b02938d1a..fcc92db7b96 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -321,11 +321,10 @@ public: }; virtual void getRemotePathsRecursive( - const String &, - std::vector &, - const std::function & /* skip_predicate */ = {}) + const String &, std::vector &, const std::function & /* skip_predicate */) { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", getDataSourceDescription().toString()); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index d7af656bea3..9f11c0ed02e 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -51,7 +51,7 @@ public: void getRemotePathsRecursive( const String & local_path, std::vector & paths_map, - const std::function & skip_predicate = {}) override; + const std::function & skip_predicate) override; const std::string & getCacheName() const override { return object_storage->getCacheName(); } diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 708c1369965..a6263f18492 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -58,8 +58,8 @@ Pipe StorageSystemRemoteDataPaths::read( if (disk->isRemote()) { std::vector remote_paths_by_local_path; - disk->getRemotePathsRecursive("store", remote_paths_by_local_path); - disk->getRemotePathsRecursive("data", remote_paths_by_local_path); + disk->getRemotePathsRecursive("store", remote_paths_by_local_path, /* skip_predicate = */ {}); + disk->getRemotePathsRecursive("data", remote_paths_by_local_path, /* skip_predicate = */ {}); if (context->getSettingsRef().traverse_shadow_remote_data_paths) disk->getRemotePathsRecursive( "shadow", diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index 2905d7801ca..a1d4b9bba46 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,9 +26,4 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true -${CLICKHOUSE_CLIENT} --query " - SELECT count() == 0 - FROM system.remote_data_paths - WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' - SETTINGS traverse_shadow_remote_data_paths=1;" +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null From df80c8c9f6ee0939cc6e6e05f3e951511a20f476 Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 10:43:48 +0300 Subject: [PATCH 037/332] Update test reference --- .../03000_traverse_shadow_system_data_paths.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference index e8183f05f5d..6ed281c757a 100644 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.reference @@ -1,3 +1,2 @@ 1 1 -1 From e789d15948eaec3eaa9a8604e24d2f6ed7b60db5 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 5 Mar 2024 16:06:25 +0800 Subject: [PATCH 038/332] optimize insertmanyfrom of nullable(number) or nullable(string) --- src/Columns/ColumnDecimal.h | 7 +++++++ src/Columns/ColumnNullable.cpp | 8 ++++++++ src/Columns/ColumnNullable.h | 1 + src/Columns/ColumnString.cpp | 21 +++++++++++++++++++++ src/Columns/ColumnString.h | 2 ++ 5 files changed, 39 insertions(+) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 7ca01a8342c..e0ea26744dc 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -56,6 +56,13 @@ public: void shrinkToFit() override { data.shrink_to_fit(); } void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast(src).getData()[n]); } + + void insertManyFrom(const IColumn & src, size_t position, size_t length) override + { + ValueType v = assert_cast(src).getData()[position]; + data.resize_fill(data.size() + length, v); + } + void insertData(const char * src, size_t /*length*/) override; void insertDefault() override { data.push_back(T()); } void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 1d11827ac97..fa5fdfb8c21 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -231,6 +231,14 @@ void ColumnNullable::insertFrom(const IColumn & src, size_t n) getNullMapData().push_back(src_concrete.getNullMapData()[n]); } + +void ColumnNullable::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnNullable & src_concrete = assert_cast(src); + getNestedColumn().insertManyFrom(src_concrete.getNestedColumn(), position, length); + getNullMapColumn().insertManyFrom(src_concrete.getNullMapColumn(), position, length); +} + void ColumnNullable::insertFromNotNullable(const IColumn & src, size_t n) { getNestedColumn().insertFrom(src, n); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index b4aef8e08fa..ef4bf4fa41b 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -69,6 +69,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertFromNotNullable(const IColumn & src, size_t n); void insertRangeFromNotNullable(const IColumn & src, size_t start, size_t length); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index b9128372cea..f3c7ac1bf0c 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -38,6 +38,27 @@ ColumnString::ColumnString(const ColumnString & src) last_offset, chars.size()); } +void ColumnString::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnString & src_concrete = assert_cast(src); + const UInt8 * src_buf = &src_concrete.chars[src_concrete.offsets[position - 1]]; + const size_t src_buf_size + = src_concrete.offsets[position] - src_concrete.offsets[position - 1]; /// -1th index is Ok, see PaddedPODArray. + + const size_t old_size = chars.size(); + const size_t new_size = old_size + src_buf_size * length; + chars.resize(new_size); + + const size_t old_rows = offsets.size(); + offsets.resize(old_rows + length); + + for (size_t current_offset = old_size; current_offset < new_size; current_offset += src_buf_size) + memcpySmallAllowReadWriteOverflow15(&chars[current_offset], src_buf, src_buf_size); + + for (size_t i = 0, current_offset = old_size + src_buf_size; i < length; ++i, current_offset += src_buf_size) + offsets[old_rows + i] = current_offset; +} + MutableColumnPtr ColumnString::cloneResized(size_t to_size) const { diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 04aa1849187..2d1d69ced73 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -160,6 +160,8 @@ public: } } + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; + void insertData(const char * pos, size_t length) override { const size_t old_size = chars.size(); From 580fd4ba080df6e29c59b785b1fca0eea76e649c Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Tue, 5 Mar 2024 10:43:48 +0300 Subject: [PATCH 039/332] Update test reference --- .../0_stateless/03000_traverse_shadow_system_data_paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh index a1d4b9bba46..98575540923 100755 --- a/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh +++ b/tests/queries/0_stateless/03000_traverse_shadow_system_data_paths.sh @@ -26,4 +26,4 @@ ${CLICKHOUSE_CLIENT} --query " FROM system.remote_data_paths WHERE disk_name = 's3_cache' AND local_path LIKE '%shadow/${BACKUP}%' SETTINGS traverse_shadow_remote_data_paths=1;" -${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null +${CLICKHOUSE_CLIENT} --query "SYSTEM UNFREEZE WITH NAME '${BACKUP}';" &>/dev/null || true From a109952960acac12790cffde030062ec60208994 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 5 Mar 2024 22:08:36 +0800 Subject: [PATCH 040/332] dev columnstring --- src/Columns/ColumnArray.cpp | 83 +++++++++++++++++++++++++++++++ src/Columns/ColumnArray.h | 9 ++++ src/Columns/ColumnConst.h | 2 + src/Columns/ColumnFixedString.cpp | 14 ++++++ src/Columns/ColumnFixedString.h | 2 + 5 files changed, 110 insertions(+) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 7b268b80116..b620da81ae8 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -347,6 +347,89 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n) getOffsets().push_back(getOffsets().back() + size); } +template +void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, size_t length) +{ + using ColVecType = ColumnVectorOrDecimal; + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + + const typename ColVecType::Container & src_data = typeid_cast(src.getData()).getData(); + typename ColVecType::Container & data_ref = typeid_cast(getData()).getData(); + size_t old_size = data_ref.size(); + size_t new_size = old_size + src_size * length; + data_ref.resize(new_size); + for (size_t i = 0, offset = old_size; i < length; ++i, offset += src_size) + memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); +} + +void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) +{ + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + + const auto & src_string = typeid_cast(src.getData()); + const auto & src_chars = src_string.getChars(); + const auto & src_string_offsets = src_string.getOffsets(); + auto & dst_string = typeid_cast(getData()); + auto & dst_chars = dst_string.getChars(); + auto & dst_string_offsets = dst_string.getOffsets(); + + /// Each row may have multiple strings, copy them to dst_chars and update dst_offsets + size_t old_size = dst_string_offsets.size(); + size_t new_size = old_size + src_size * length; + dst_string_offsets.resize(new_size); + size_t dst_string_offset = dst_chars.size(); + for (size_t i = 0; i < length; ++i) + { + for (size_t j = 0; j < src_size; ++j) + { + size_t nested_offset = src_string_offsets[src_offset + j - 1]; + size_t nested_length = src_string_offsets[src_offset + j] - nested_offset; + + dst_string_offset += nested_length; + dst_string_offsets[old_size + i * src_size + j] = dst_string_offset; + } + } + + size_t chars_to_copy = src_string_offsets[src_offset + src_size - 1] - src_string_offsets[src_offset - 1]; + dst_chars.resize(dst_chars.size() + chars_to_copy * length); + for (size_t dst_offset = old_size; dst_offset < new_size; dst_offset += src_size) + memcpy(&dst_chars[dst_string_offsets[dst_offset - 1]], &src_chars[src_string_offsets[src_offset - 1]], chars_to_copy); +} + +void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) +{ + +} +void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) +{ + +} +void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) +{ + size_t src_size = src.sizeAt(position); + size_t src_offset = src.offsetAt(position); + const auto & src_data = src.getData(); + size_t new_size = data->size() + src_size * length; + data->reserve(new_size); + for (size_t i = 0; i < length; ++i) + data->insertRangeFrom(src_data, src_offset, src_size); +} + +void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) +{ + /// First fill offsets + const ColumnArray & src = assert_cast(src_); + size_t src_size = src.sizeAt(position); + auto & offsets_ref = getOffsets(); + size_t old_rows = offsets_ref.size(); + size_t new_rows = old_rows + length; + size_t old_size = offsets_ref.back(); + offsets_ref.resize(new_rows); + for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) + offsets_ref[old_rows + i] = offset; +} void ColumnArray::insertDefault() { diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 230d8830265..73d632a38b9 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -88,6 +88,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; @@ -213,6 +214,14 @@ private: ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; + /// Specializations for insertManyFrom + template + void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); + void insertManyFromString(const ColumnArray & src, size_t position, size_t length); + void insertManyFromTuple(const ColumnArray & src, size_t position, size_t length); + void insertManyFromNullable(const ColumnArray & src, size_t position, size_t length); + void insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length); + int compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint, const Collator * collator=nullptr) const; }; diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 990b7189fa3..4a3d40ca0d2 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -150,6 +150,8 @@ public: ++s; } + void insertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; } + void insertDefault() override { ++s; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index e460c84d696..b55f68d4687 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -85,6 +85,20 @@ void ColumnFixedString::insertFrom(const IColumn & src_, size_t index) memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n); } +void ColumnFixedString::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnFixedString & src_concrete = assert_cast(src); + if (n != src_concrete.getN()) + throw Exception(ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, "Size of FixedString doesn't match"); + + const size_t old_size = chars.size(); + const size_t new_size = old_size + n * length; + chars.resize(new_size); + + for (size_t offset = old_size; offset < new_size; offset += n) + memcpySmallAllowReadWriteOverflow15(&chars[offset], &src_concrete.chars[n * position], n); +} + void ColumnFixedString::insertData(const char * pos, size_t length) { if (length > n) diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index f40e1356b27..56d42e8b34e 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -100,6 +100,8 @@ public: void insertFrom(const IColumn & src_, size_t index) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; + void insertData(const char * pos, size_t length) override; void insertDefault() override From 53c9d4513c4b93ed79df305bb5c36c0cfb43ef79 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 12:16:17 +0800 Subject: [PATCH 041/332] finish dev column array --- src/Columns/ColumnArray.cpp | 132 +++++++++++++++++++++++++++++++++--- src/Columns/ColumnArray.h | 3 + 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index b620da81ae8..aa0d5aa3e50 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -31,6 +31,7 @@ namespace ErrorCodes extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int LOGICAL_ERROR; extern const int TOO_LARGE_ARRAY_SIZE; + extern const int ILLEGAL_COLUMN; } /** Obtaining array as Field can be slow for large arrays and consume vast amount of memory. @@ -363,6 +364,19 @@ void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); } +void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t position, size_t length) +{ + const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); + if (!src_array) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot insert from const column of type {} to column of type {}", + src.getDataColumn().getName(), + getName()); + + insertManyFromImpl(*src_array, 0, length, true); +} + void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) { size_t src_size = src.sizeAt(position); @@ -400,12 +414,53 @@ void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) { + ColumnTuple & tuple = assert_cast(getData()); + const ColumnTuple & src_tuple = assert_cast(src.getData()); + /// Make temporary arrays for each components of Tuple. In the same way as for Nullable. + size_t tuple_size = tuple.tupleSize(); + size_t src_tuple_size = src_tuple.tupleSize(); + if (tuple_size == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty tuple"); + if (tuple_size != src_tuple_size) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); + + Columns temporary_arrays(tuple_size); + Columns src_temporary_arrays(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + { + temporary_arrays[i] = ColumnArray::create(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable()); + src_temporary_arrays[i] = ColumnArray::create(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + assert_cast(*temporary_arrays[i]) + .insertManyFromImpl(assert_cast(*src_temporary_arrays[i]), position, length, false); + } + + Columns tuple_columns(tuple_size); + for (size_t i = 0; i < tuple_size; ++i) + tuple_columns[i] = assert_cast(*temporary_arrays[i]).getDataPtr(); + + getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); } + void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) { + ColumnNullable & nullable = assert_cast(getData()); + const ColumnNullable & src_nullable = assert_cast(src.getData()); + /// Process nested column without updating array offsets + auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); + + /// Process null map column without updating array offsets + auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); + + /// Update array data + getDataPtr() = ColumnNullable::create(array_of_nested.getDataPtr(), array_of_null_map.getDataPtr()); } + void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) { size_t src_size = src.sizeAt(position); @@ -419,16 +474,73 @@ void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) { - /// First fill offsets - const ColumnArray & src = assert_cast(src_); - size_t src_size = src.sizeAt(position); - auto & offsets_ref = getOffsets(); - size_t old_rows = offsets_ref.size(); - size_t new_rows = old_rows + length; - size_t old_size = offsets_ref.back(); - offsets_ref.resize(new_rows); - for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) - offsets_ref[old_rows + i] = offset; + const ColumnConst * src_const = typeid_cast(&src_); + if (src_const) + return insertManyFromConst(*src_const, position, length); + + const ColumnArray * src_array = typeid_cast(&src_); + if (!src_array) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert from column of type {} to column of type {}", src_.getName(), getName()); + + return insertManyFromImpl(*src_array, position, length, true); +} + +void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets) +{ + /// First fill offsets if needed + if (update_offsets) + { + size_t src_size = src.sizeAt(position); + auto & offsets_ref = getOffsets(); + size_t old_rows = offsets_ref.size(); + size_t new_rows = old_rows + length; + size_t old_size = offsets_ref.back(); + offsets_ref.resize(new_rows); + for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) + offsets_ref[old_rows + i] = offset; + } + + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast *>(data.get())) + return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromNullable(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromTuple(src, position, length); + return insertManyFromGeneric(src, position, length); } void ColumnArray::insertDefault() diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 73d632a38b9..765f86ec552 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -215,6 +215,9 @@ private: ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; /// Specializations for insertManyFrom + void insertManyFromConst(const ColumnConst & src, size_t position, size_t length); + void insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets = true); + template void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); void insertManyFromString(const ColumnArray & src, size_t position, size_t length); From 3bf3c7cc708d1a564896d649a1a804b868f89d8d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 12:32:23 +0800 Subject: [PATCH 042/332] finish column map and tuple --- src/Columns/ColumnArray.cpp | 2 +- src/Columns/ColumnMap.cpp | 5 +++++ src/Columns/ColumnMap.h | 1 + src/Columns/ColumnTuple.cpp | 12 ++++++++++++ src/Columns/ColumnTuple.h | 1 + 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index aa0d5aa3e50..5b0df8e9b6b 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -364,7 +364,7 @@ void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); } -void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t position, size_t length) +void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t /*position*/, size_t length) { const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); if (!src_array) diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index 995f3103484..57e8ba685b4 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -158,6 +158,11 @@ void ColumnMap::insertFrom(const IColumn & src, size_t n) nested->insertFrom(assert_cast(src).getNestedColumn(), n); } +void ColumnMap::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + assert_cast(*nested).insertManyFrom(assert_cast(src).getNestedColumn(), position, length); +} + void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length) { nested->insertRangeFrom( diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 17cd86a3788..60aa69e7bf6 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -67,6 +67,7 @@ public: void updateWeakHash32(WeakHash32 & hash) const override; void updateHashFast(SipHash & hash) const override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; void expand(const Filter & mask, bool inverted) override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 17cc58d92f5..062bdadf9d2 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -185,6 +185,18 @@ void ColumnTuple::insertFrom(const IColumn & src_, size_t n) columns[i]->insertFrom(*src.columns[i], n); } +void ColumnTuple::insertManyFrom(const IColumn & src, size_t position, size_t length) +{ + const ColumnTuple & src_tuple = assert_cast(src); + + const size_t tuple_size = columns.size(); + if (src_tuple.columns.size() != tuple_size) + throw Exception(ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE, "Cannot insert value of different size into tuple"); + + for (size_t i = 0; i < tuple_size; ++i) + columns[i]->insertManyFrom(*src_tuple.columns[i], position, length); +} + void ColumnTuple::insertDefault() { for (auto & column : columns) diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 610416b8b11..5b626155754 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -60,6 +60,7 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; + void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override; From 3005bff23100539dbb71f9623dc3aed9c34a87f6 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 14:43:33 +0800 Subject: [PATCH 043/332] fix building --- src/Columns/ColumnArray.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 5b0df8e9b6b..389b3e97820 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -425,7 +425,7 @@ void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, if (tuple_size != src_tuple_size) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - Columns temporary_arrays(tuple_size); + MutableColumns temporary_arrays(tuple_size); Columns src_temporary_arrays(tuple_size); for (size_t i = 0; i < tuple_size; ++i) { From 8e413da8f156ab03c875b9525044265cffcc5b83 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 6 Mar 2024 17:32:08 +0800 Subject: [PATCH 044/332] apply opts for string nested in array --- src/Columns/ColumnArray.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 389b3e97820..44b17c89ae1 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -536,6 +536,8 @@ void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, s return insertManyFromNumber(src, position, length); if (typeid_cast *>(data.get())) return insertManyFromNumber(src, position, length); + if (typeid_cast(data.get())) + return insertManyFromString(src, position, length); if (typeid_cast(data.get())) return insertManyFromNullable(src, position, length); if (typeid_cast(data.get())) From 2dc1721262c9f483917750aaa6139ff7409e02dc Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 6 Mar 2024 11:53:00 +0000 Subject: [PATCH 045/332] Refactorings for consistency --- src/Functions/array/arrayDistance.cpp | 54 +++++++-------- src/Functions/array/arrayDotProduct.cpp | 91 +++++++++++++------------ 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 71564f6fa93..6b72c99d829 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -379,17 +379,17 @@ public: } -#define SUPPORTED_TYPES(action) \ - action(UInt8) \ - action(UInt16) \ - action(UInt32) \ - action(UInt64) \ - action(Int8) \ - action(Int16) \ - action(Int32) \ - action(Int64) \ - action(Float32) \ - action(Float64) +#define SUPPORTED_TYPES(ACTION) \ + ACTION(UInt8) \ + ACTION(UInt16) \ + ACTION(UInt32) \ + ACTION(UInt64) \ + ACTION(Int8) \ + ACTION(Int16) \ + ACTION(Int32) \ + ACTION(Int64) \ + ACTION(Float32) \ + ACTION(Float64) private: @@ -398,12 +398,11 @@ private: { DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); - /// Dynamic disaptch based on the 1st argument type switch (type_x->getTypeId()) { #define ON_TYPE(type) \ case TypeIndex::type: \ - return executeWithFirstType(arguments, input_rows_count); \ + return executeWithResultTypeAndLeftType(arguments, input_rows_count); \ break; SUPPORTED_TYPES(ON_TYPE) @@ -419,17 +418,16 @@ private: } } - template - ColumnPtr executeWithFirstType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + template + ColumnPtr executeWithResultTypeAndLeftType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); - /// Dynamic disaptch based on the 2nd argument type switch (type_y->getTypeId()) { #define ON_TYPE(type) \ case TypeIndex::type: \ - return executeWithTypes(arguments[0].column, arguments[1].column, input_rows_count, arguments); \ + return executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column, input_rows_count, arguments); \ break; SUPPORTED_TYPES(ON_TYPE) @@ -445,16 +443,16 @@ private: } } - template - ColumnPtr executeWithTypes(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count, const ColumnsWithTypeAndName & arguments) const + template + ColumnPtr executeWithResultTypeAndLeftTypeAndRightType(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count, const ColumnsWithTypeAndName & arguments) const { if (typeid_cast(col_x.get())) { - return executeWithTypesFirstArgConst(col_x, col_y, input_rows_count, arguments); + return executeWithLeftArgConst(col_x, col_y, input_rows_count, arguments); } else if (typeid_cast(col_y.get())) { - return executeWithTypesFirstArgConst(col_y, col_x, input_rows_count, arguments); + return executeWithLeftArgConst(col_y, col_x, input_rows_count, arguments); } col_x = col_x->convertToFullColumnIfConst(); @@ -463,8 +461,8 @@ private: const auto & array_x = *assert_cast(col_x.get()); const auto & array_y = *assert_cast(col_y.get()); - const auto & data_x = typeid_cast &>(array_x.getData()).getData(); - const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); const auto & offsets_x = array_x.getOffsets(); const auto & offsets_y = array_y.getOffsets(); @@ -521,8 +519,8 @@ private: } /// Special case when the 1st parameter is Const - template - ColumnPtr executeWithTypesFirstArgConst(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count, const ColumnsWithTypeAndName & arguments) const + template + ColumnPtr executeWithLeftArgConst(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count, const ColumnsWithTypeAndName & arguments) const { col_x = assert_cast(col_x.get())->getDataColumnPtr(); col_y = col_y->convertToFullColumnIfConst(); @@ -530,8 +528,8 @@ private: const auto & array_x = *assert_cast(col_x.get()); const auto & array_y = *assert_cast(col_y.get()); - const auto & data_x = typeid_cast &>(array_x.getData()).getData(); - const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); const auto & offsets_x = array_x.getOffsets(); const auto & offsets_y = array_y.getOffsets(); @@ -574,7 +572,7 @@ private: /// - the two most common metrics L2 and cosine distance, /// - the most powerful SIMD instruction set (AVX-512F). #if USE_MULTITARGET_CODE - if constexpr (std::is_same_v && std::is_same_v) /// ResultType is Float32 or Float64 + if constexpr (std::is_same_v && std::is_same_v) /// ResultType is Float32 or Float64 { if constexpr (std::is_same_v || std::is_same_v) diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 6c615a058c3..548c79c567f 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -163,26 +163,29 @@ public: return Kernel::getReturnType(nested_types[0], nested_types[1]); } +#define SUPPORTED_TYPES(ACTION) \ + ACTION(UInt8) \ + ACTION(UInt16) \ + ACTION(UInt32) \ + ACTION(UInt64) \ + ACTION(Int8) \ + ACTION(Int16) \ + ACTION(Int32) \ + ACTION(Int64) \ + ACTION(Float32) \ + ACTION(Float64) + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override { switch (result_type->getTypeId()) { - #define SUPPORTED_TYPE(type) \ + #define ON_TYPE(type) \ case TypeIndex::type: \ return executeWithResultType(arguments); \ break; - SUPPORTED_TYPE(UInt8) - SUPPORTED_TYPE(UInt16) - SUPPORTED_TYPE(UInt32) - SUPPORTED_TYPE(UInt64) - SUPPORTED_TYPE(Int8) - SUPPORTED_TYPE(Int16) - SUPPORTED_TYPE(Int32) - SUPPORTED_TYPE(Int64) - SUPPORTED_TYPE(Float32) - SUPPORTED_TYPE(Float64) - #undef SUPPORTED_TYPE + SUPPORTED_TYPES(ON_TYPE) + #undef ON_TYPE default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type {}", result_type->getName()); @@ -194,16 +197,16 @@ private: ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments) const { ColumnPtr res; - if (!((res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)) - || (res = executeWithResultTypeAndLeft(arguments)))) + if (!((res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)) + || (res = executeWithResultTypeAndLeftType(arguments)))) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); @@ -211,43 +214,43 @@ private: } template - ColumnPtr executeWithResultTypeAndLeft(const ColumnsWithTypeAndName & arguments) const + ColumnPtr executeWithResultTypeAndLeftType(const ColumnsWithTypeAndName & arguments) const { ColumnPtr res; - if ( (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments)) - || (res = executeWithResultTypeAndLeftAndRight(arguments))) + if ( (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) + || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column))) return res; return nullptr; } template - ColumnPtr executeWithResultTypeAndLeftAndRight(const ColumnsWithTypeAndName & arguments) const + ColumnPtr executeWithResultTypeAndLeftTypeAndRightType(ColumnPtr col_x, ColumnPtr col_y) const { - ColumnPtr col_left = arguments[0].column->convertToFullColumnIfConst(); - ColumnPtr col_right = arguments[1].column->convertToFullColumnIfConst(); - if (!col_left || !col_right) + col_x = col_x->convertToFullColumnIfConst(); + col_y = col_y->convertToFullColumnIfConst(); + if (!col_x || !col_y) return nullptr; - const ColumnArray * col_arr_left = checkAndGetColumn(col_left.get()); - const ColumnArray * cokl_arr_right = checkAndGetColumn(col_right.get()); - if (!col_arr_left || !cokl_arr_right) + const ColumnArray * array_x = checkAndGetColumn(col_x.get()); + const ColumnArray * array_y = checkAndGetColumn(col_y.get()); + if (!array_x || !array_y) return nullptr; - const ColumnVector * col_arr_nested_left = checkAndGetColumn>(col_arr_left->getData()); - const ColumnVector * col_arr_nested_right = checkAndGetColumn>(cokl_arr_right->getData()); + const ColumnVector * col_arr_nested_left = checkAndGetColumn>(array_x->getData()); + const ColumnVector * col_arr_nested_right = checkAndGetColumn>(array_y->getData()); if (!col_arr_nested_left || !col_arr_nested_right) return nullptr; - if (!col_arr_left->hasEqualOffsets(*cokl_arr_right)) + if (!array_x->hasEqualOffsets(*array_y)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName()); auto col_res = ColumnVector::create(); @@ -255,7 +258,7 @@ private: vector( col_arr_nested_left->getData(), col_arr_nested_right->getData(), - col_arr_left->getOffsets(), + array_x->getOffsets(), col_res->getData()); return col_res; From 076482e8bd503ca352695173d87e9b48228389f0 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 6 Mar 2024 14:04:09 +0100 Subject: [PATCH 046/332] Remove whitespaces --- src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 42c6e63da01..e0d0fda81cb 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -132,7 +132,7 @@ class OptimizeShardingKeyRewriteIn : public InDepthQueryTreeVisitorWithContext; - + OptimizeShardingKeyRewriteIn(OptimizeShardingKeyRewriteInVisitor::Data data_, ContextPtr context) : Base(std::move(context)) , data(std::move(data_)) From 7065e650e1d007be4659ddb1f070b48e19cdef55 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 6 Mar 2024 14:34:03 +0100 Subject: [PATCH 047/332] Code cleanup --- src/DataTypes/ObjectUtils.cpp | 24 ++------ src/DataTypes/ObjectUtils.h | 2 - .../ClusterProxy/SelectStreamFactory.cpp | 7 +-- .../ClusterProxy/SelectStreamFactory.h | 2 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 61 +++---------------- src/Storages/StorageDistributed.cpp | 1 - tests/analyzer_integration_broken_tests.txt | 1 - .../test_distributed_type_object/test.py | 3 +- 8 files changed, 18 insertions(+), 83 deletions(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 47d8c5c9113..ccfa0a28f13 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1,4 +1,8 @@ #include +#include +#include +#include +#include #include #include #include @@ -21,16 +25,6 @@ #include #include #include -#include "Analyzer/ConstantNode.h" -#include "Analyzer/FunctionNode.h" -#include "Analyzer/IQueryTreeNode.h" -#include "Analyzer/Identifier.h" -#include "Analyzer/IdentifierNode.h" -#include "Analyzer/QueryNode.h" -#include "Analyzer/Utils.h" -#include -#include -#include "Common/logger_useful.h" namespace DB @@ -991,22 +985,12 @@ MissingObjectList replaceMissedSubcolumnsByConstants( { auto constant = std::make_shared(type->getDefault(), type); constant->setAlias(table_expression->getAlias() + "." + name); - // auto materialize = std::make_shared("materialize"); - - // auto function = FunctionFactory::instance().get("materialize", context); - // materialize->getArguments().getNodes() = { constant }; - // materialize->resolveAsFunction(function->build(materialize->getArgumentColumns())); - // materialize->setAlias(name); column_name_to_node[name] = buildCastFunction(constant, type, context); missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "{} -> {}", missed_list.back().first, missed_list.back().second); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Name {} Expression\n{}", name, column_name_to_node[name]->dumpTree()); } - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Table expression\n{} ", table_expression->dumpTree()); replaceColumns(query, table_expression, column_name_to_node); - LOG_DEBUG(&Poco::Logger::get("replaceMissedSubcolumnsByConstants"), "Result:\n{} ", query->dumpTree()); return missed_list; } diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 013e525832e..6ef19baf5ae 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -3,8 +3,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" -#include "Interpreters/Context_fwd.h" #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5bcd1ce68cb..4fccd83c8c0 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,10 +5,9 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" -#include "Interpreters/InterpreterSelectQueryAnalyzer.h" -#include "Interpreters/SelectQueryOptions.h" -#include "Planner/Utils.h" +#include +#include +#include #include #include #include diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index bee7edb3c19..61694830b3d 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" namespace DB { diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index ac507c6d555..72848a37f6e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -15,7 +14,6 @@ #include #include #include -#include "DataTypes/ObjectUtils.h" #include #include #include @@ -33,54 +31,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static void addRenamingActions(Pipe & pipe, const MissingObjectList & missed_list, const Block & output_header) -{ - if (missed_list.empty()) - return; - - const auto & output_columns = output_header.getColumnsWithTypeAndName(); - std::vector indexes; - for (size_t i = 0; i < output_columns.size(); ++i) - { - bool found = false; - for (auto const & elem : missed_list) - { - if (output_columns[i].name.contains(elem.second)) - { - found = true; - break; - } - } - if (found) - indexes.push_back(i); - } - - auto dag = std::make_shared(pipe.getHeader().getColumnsWithTypeAndName()); - - for (size_t index : indexes) - { - dag->addOrReplaceInOutputs(dag->addAlias(*dag->getOutputs()[index], output_header.getByPosition(index).name)); - } - - // dag->addAliases(rename_to_apply); - - auto convert_actions = std::make_shared(dag); - pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr - { - return std::make_shared(cur_header, convert_actions); - }); - - LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "EXPECTED:\n{}", output_header.dumpStructure()); - - LOG_DEBUG(&Poco::Logger::get("addRenamingActions"), "{}", pipe.getHeader().dumpStructure()); -} - -static void addConvertingActions(Pipe & pipe, const Block & header) +static void addConvertingActions(Pipe & pipe, const Block & header, bool use_positions_to_match = false) { if (blocksHaveEqualStructure(pipe.getHeader(), header)) return; - auto get_converting_dag = [](const Block & block_, const Block & header_) + auto match_mode = use_positions_to_match ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode = match_mode](const Block & block_, const Block & header_) { /// Convert header structure to expected. /// Also we ignore constants from result and replace it with constants from header. @@ -88,7 +46,7 @@ static void addConvertingActions(Pipe & pipe, const Block & header) return ActionsDAG::makeConvertingActions( block_.getColumnsWithTypeAndName(), header_.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name, + mode, true); }; @@ -260,8 +218,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -342,8 +299,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } } else @@ -372,8 +328,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addRenamingActions(pipes.back(), shard.missing_object_list, output_stream->header); - addConvertingActions(pipes.back(), output_stream->header); + addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); } } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 34ab21a4751..726f1788115 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -30,7 +30,6 @@ #include #include #include -#include "Analyzer/IQueryTreeNode.h" #include #include diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt index 796ca6bca22..a7954f91efa 100644 --- a/tests/analyzer_integration_broken_tests.txt +++ b/tests/analyzer_integration_broken_tests.txt @@ -1,6 +1,5 @@ test_build_sets_from_multiple_threads/test.py::test_set test_concurrent_backups_s3/test.py::test_concurrent_backups -test_distributed_type_object/test.py::test_distributed_type_object test_merge_table_over_distributed/test.py::test_global_in test_merge_table_over_distributed/test.py::test_select_table_name_from_merge_over_distributed test_passing_max_partitions_to_read_remotely/test.py::test_default_database_on_cluster diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py index 7e6c000cb8e..360087c9dda 100644 --- a/tests/integration/test_distributed_type_object/test.py +++ b/tests/integration/test_distributed_type_object/test.py @@ -85,10 +85,11 @@ def test_distributed_type_object(started_cluster): 3\t\t\t\tfoo""" ) + # The following query is not supported by analyzer now assert ( TSV( node1.query( - "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id" + "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id SETTINGS allow_experimental_analyzer = 0" ) ) == expected From c3909743ed95adf5efd16e69f353aab8af73978b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 6 Mar 2024 12:36:00 +0000 Subject: [PATCH 048/332] Remove repeated unnecessary unpacking of const columns --- src/Functions/array/arrayDotProduct.cpp | 88 +++++++++++++------------ 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 548c79c567f..c27170cd35b 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -19,7 +19,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; } @@ -196,40 +195,51 @@ private: template ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments) const { - ColumnPtr res; - if (!((res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)) - || (res = executeWithResultTypeAndLeftType(arguments)))) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); + DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); - return res; + switch (type_x->getTypeId()) + { +#define ON_TYPE(type) \ + case TypeIndex::type: \ + return executeWithResultTypeAndLeftType(arguments); \ + break; + + SUPPORTED_TYPES(ON_TYPE) +#undef ON_TYPE + + default: + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), + type_x->getName()); + } } template ColumnPtr executeWithResultTypeAndLeftType(const ColumnsWithTypeAndName & arguments) const { - ColumnPtr res; - if ( (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column)) - || (res = executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column))) - return res; + DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); - return nullptr; + switch (type_y->getTypeId()) + { + #define ON_TYPE(type) \ + case TypeIndex::type: \ + return executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column); \ + break; + + SUPPORTED_TYPES(ON_TYPE) + #undef ON_TYPE + + default: + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), + type_y->getName()); + } } template @@ -237,28 +247,22 @@ private: { col_x = col_x->convertToFullColumnIfConst(); col_y = col_y->convertToFullColumnIfConst(); - if (!col_x || !col_y) - return nullptr; - const ColumnArray * array_x = checkAndGetColumn(col_x.get()); - const ColumnArray * array_y = checkAndGetColumn(col_y.get()); - if (!array_x || !array_y) - return nullptr; + const auto & array_x = *assert_cast(col_x.get()); + const auto & array_y = *assert_cast(col_y.get()); - const ColumnVector * col_arr_nested_left = checkAndGetColumn>(array_x->getData()); - const ColumnVector * col_arr_nested_right = checkAndGetColumn>(array_y->getData()); - if (!col_arr_nested_left || !col_arr_nested_right) - return nullptr; + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); - if (!array_x->hasEqualOffsets(*array_y)) + if (!array_x.hasEqualOffsets(array_y)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName()); auto col_res = ColumnVector::create(); vector( - col_arr_nested_left->getData(), - col_arr_nested_right->getData(), - array_x->getOffsets(), + data_x, + data_y, + array_x.getOffsets(), col_res->getData()); return col_res; From a69bcc29f5bb0bdaca1757673dac1574c97b1e2f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 6 Mar 2024 14:53:11 +0000 Subject: [PATCH 049/332] Refactor a sanity check --- src/Functions/array/arrayDistance.cpp | 20 +++---------------- src/Functions/array/arrayDotProduct.cpp | 8 +++++--- .../queries/0_stateless/02708_dotProduct.sql | 2 +- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 6b72c99d829..c9e6e97749f 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -18,11 +18,11 @@ namespace DB { namespace ErrorCodes { + extern const int ARGUMENT_OUT_OF_BOUND; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; extern const int SIZES_OF_ARRAYS_DONT_MATCH; - extern const int ARGUMENT_OUT_OF_BOUND; } struct L1Distance @@ -465,22 +465,9 @@ private: const auto & data_y = typeid_cast &>(array_y.getData()).getData(); const auto & offsets_x = array_x.getOffsets(); - const auto & offsets_y = array_y.getOffsets(); - /// Check that arrays in both columns are the sames size - for (size_t row = 0; row < offsets_x.size(); ++row) - { - if (offsets_x[row] != offsets_y[row]) [[unlikely]] - { - ColumnArray::Offset prev_offset = row > 0 ? offsets_x[row] : 0; - throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, - "Arguments of function {} have different array sizes: {} and {}", - getName(), - offsets_x[row] - prev_offset, - offsets_y[row] - prev_offset); - } - } + if (!array_x.hasEqualOffsets(array_y)) + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Array arguments for function {} must have equal sizes", getName()); const typename Kernel::ConstParams kernel_params = initConstParams(arguments); @@ -534,7 +521,6 @@ private: const auto & offsets_x = array_x.getOffsets(); const auto & offsets_y = array_y.getOffsets(); - /// Check that arrays in both columns are the sames size ColumnArray::Offset prev_offset = 0; for (size_t row : collections::range(0, offsets_y.size())) { diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index c27170cd35b..3abd1a6c6db 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -18,9 +18,9 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; + extern const int SIZES_OF_ARRAYS_DONT_MATCH; } @@ -254,15 +254,17 @@ private: const auto & data_x = typeid_cast &>(array_x.getData()).getData(); const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + const auto & offsets_x = array_x.getOffsets(); + if (!array_x.hasEqualOffsets(array_y)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Array arguments for function {} must have equal sizes", getName()); + throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Array arguments for function {} must have equal sizes", getName()); auto col_res = ColumnVector::create(); vector( data_x, data_y, - array_x.getOffsets(), + offsets_x, col_res->getData()); return col_res; diff --git a/tests/queries/0_stateless/02708_dotProduct.sql b/tests/queries/0_stateless/02708_dotProduct.sql index 6ad615664e8..2035e23cf1d 100644 --- a/tests/queries/0_stateless/02708_dotProduct.sql +++ b/tests/queries/0_stateless/02708_dotProduct.sql @@ -4,7 +4,7 @@ SELECT arrayDotProduct([1, 2]); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATC SELECT arrayDotProduct([1, 2], 'abc'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayDotProduct('abc', [1, 2]); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT arrayDotProduct([1, 2], ['abc', 'def']); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT arrayDotProduct([1, 2], [3, 4, 5]); -- { serverError BAD_ARGUMENTS } +SELECT arrayDotProduct([1, 2], [3, 4, 5]); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH } SELECT dotProduct([1, 2], (3, 4, 5)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } SELECT '-- Tests'; From d9b5f9a086d8bc1af5387edee77d0e3fdbf4d9b0 Mon Sep 17 00:00:00 2001 From: Nikolay Monkov Date: Wed, 6 Mar 2024 21:37:56 +0500 Subject: [PATCH 050/332] version has been added to docker labels --- tests/ci/docker_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 38d0ea6d86b..803dbfcd92a 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -216,11 +216,12 @@ def gen_tags(version: ClickHouseVersion, release_type: str) -> List[str]: return tags -def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str]) -> List[str]: +def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str], version: str) -> List[str]: args = [ f"--platform=linux/{arch}", f"--label=build-url={GITHUB_RUN_URL}", f"--label=com.clickhouse.build.githash={git.sha}", + f"--label=com.clickhouse.build.version={version}", ] if direct_urls: args.append(f"--build-arg=DIRECT_DOWNLOAD_URLS='{' '.join(direct_urls)}'") @@ -267,7 +268,7 @@ def build_and_push_image( urls = [url for url in direct_urls[arch] if ".deb" in url] else: urls = [url for url in direct_urls[arch] if ".tgz" in url] - cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls)) + cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe)) if not push: cmd_args.append(f"--tag={image.repo}:{arch_tag}") cmd_args.extend( From 245ea0c186614a5a13a7e0bed79f94bc6ad46d87 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 6 Mar 2024 15:26:11 +0000 Subject: [PATCH 051/332] Implement const/non-const shortcut --- src/Functions/array/arrayDistance.cpp | 3 +- src/Functions/array/arrayDotProduct.cpp | 110 ++++++++++++++---- tests/performance/dotProduct.xml | 1 + .../0_stateless/02708_dotProduct.reference | 7 ++ .../queries/0_stateless/02708_dotProduct.sql | 9 ++ 5 files changed, 106 insertions(+), 24 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index c9e6e97749f..03f0bc7b286 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -477,6 +477,7 @@ private: /// Do the actual computation ColumnArray::Offset prev = 0; size_t row = 0; + for (auto off : offsets_x) { /// Process chunks in vectorized manner @@ -522,7 +523,7 @@ private: const auto & offsets_y = array_y.getOffsets(); ColumnArray::Offset prev_offset = 0; - for (size_t row : collections::range(0, offsets_y.size())) + for (size_t row = 0; row < offsets_y.size(); ++row) { if (offsets_x[0] != offsets_y[row] - prev_offset) [[unlikely]] { diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 3abd1a6c6db..f9a6687e028 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -245,6 +245,15 @@ private: template ColumnPtr executeWithResultTypeAndLeftTypeAndRightType(ColumnPtr col_x, ColumnPtr col_y) const { + if (typeid_cast(col_x.get())) + { + return executeWithLeftArgConst(col_x, col_y); + } + else if (typeid_cast(col_y.get())) + { + return executeWithLeftArgConst(col_y, col_x); + } + col_x = col_x->convertToFullColumnIfConst(); col_y = col_y->convertToFullColumnIfConst(); @@ -260,30 +269,83 @@ private: throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Array arguments for function {} must have equal sizes", getName()); auto col_res = ColumnVector::create(); + auto & result = col_res->getData(); - vector( - data_x, - data_y, - offsets_x, - col_res->getData()); - - return col_res; - } - - template - static void vector( - const PaddedPODArray & left, - const PaddedPODArray & right, - const ColumnArray::Offsets & offsets, - PaddedPODArray & result) - { - size_t size = offsets.size(); + size_t size = offsets_x.size(); result.resize(size); ColumnArray::Offset current_offset = 0; for (size_t row = 0; row < size; ++row) { - size_t array_size = offsets[row] - current_offset; + size_t array_size = offsets_x[row] - current_offset; + + size_t i = 0; + + /// Process chunks in vectorized manner + static constexpr size_t VEC_SIZE = 4; + typename Kernel::template State states[VEC_SIZE]; + for (; i + VEC_SIZE < array_size; i += VEC_SIZE) + { + for (size_t j = 0; j < VEC_SIZE; ++j) + Kernel::template accumulate(states[j], static_cast(data_x[current_offset + i + j]), static_cast(data_y[current_offset + i + j])); + } + + typename Kernel::template State state; + for (const auto & other_state : states) + Kernel::template combine(state, other_state); + + /// Process the tail + for (; i < array_size; ++i) + Kernel::template accumulate(state, static_cast(data_x[current_offset + i]), static_cast(data_y[current_offset + i])); + + result[row] = Kernel::template finalize(state); + + current_offset = offsets_x[row]; + } + + return col_res; + } + + template + ColumnPtr executeWithLeftArgConst(ColumnPtr col_x, ColumnPtr col_y) const + { + col_x = assert_cast(col_x.get())->getDataColumnPtr(); + col_y = col_y->convertToFullColumnIfConst(); + + const auto & array_x = *assert_cast(col_x.get()); + const auto & array_y = *assert_cast(col_y.get()); + + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + + const auto & offsets_x = array_x.getOffsets(); + const auto & offsets_y = array_y.getOffsets(); + + ColumnArray::Offset prev_offset = 0; + for (size_t row = 0; row < offsets_y.size(); ++row) + { + if (offsets_x[0] != offsets_y[row] - prev_offset) [[unlikely]] + { + throw Exception( + ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, + "Arguments of function {} have different array sizes: {} and {}", + getName(), + offsets_x[0], + offsets_y[row] - prev_offset); + } + prev_offset = offsets_y[row]; + } + + auto col_res = ColumnVector::create(); + auto & result = col_res->getData(); + + size_t size = offsets_y.size(); + result.resize(size); + + ColumnArray::Offset current_offset = 0; + for (size_t row = 0; row < size; ++row) + { + size_t array_size = offsets_x[0]; typename Kernel::template State state; size_t i = 0; @@ -292,13 +354,14 @@ private: /// To avoid combinatorial explosion of SIMD kernels, focus on /// - the two most common input/output types (Float32 x Float32) --> Float32 and (Float64 x Float64) --> Float64 instead of 10 x /// 10 input types x 8 output types, + /// - const/non-const inputs instead of non-const/non-const inputs /// - the most powerful SIMD instruction set (AVX-512F). #if USE_MULTITARGET_CODE if constexpr ((std::is_same_v || std::is_same_v) && std::is_same_v && std::is_same_v) { if (isArchSupported(TargetArch::AVX512F)) - Kernel::template accumulateCombine(&left[current_offset], &right[current_offset], array_size, i, state); + Kernel::template accumulateCombine(&data_x[0], &data_y[current_offset], array_size, i, state); } #else /// Process chunks in vectorized manner @@ -307,7 +370,7 @@ private: for (; i + VEC_SIZE < array_size; i += VEC_SIZE) { for (size_t j = 0; j < VEC_SIZE; ++j) - Kernel::template accumulate(states[j], static_cast(left[i + j]), static_cast(right[i + j])); + Kernel::template accumulate(states[j], static_cast(data_x[i + j]), static_cast(data_y[current_offset + i + j])); } for (const auto & other_state : states) @@ -316,13 +379,14 @@ private: /// Process the tail for (; i < array_size; ++i) - Kernel::template accumulate(state, static_cast(left[i]), static_cast(right[i])); + Kernel::template accumulate(state, static_cast(data_x[i]), static_cast(data_y[current_offset + i])); - /// ResultType res = Kernel::template finalize(state); result[row] = Kernel::template finalize(state); - current_offset = offsets[row]; + current_offset = offsets_y[row]; } + + return col_res; } }; diff --git a/tests/performance/dotProduct.xml b/tests/performance/dotProduct.xml index 6e056964ebb..a0ab7beea9c 100644 --- a/tests/performance/dotProduct.xml +++ b/tests/performance/dotProduct.xml @@ -56,6 +56,7 @@ SELECT sum(dp) FROM (SELECT dotProduct(v, v) AS dp FROM vecs_{element_type}) + WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dp) FROM (SELECT dotProduct(a, v) AS dp FROM vecs_{element_type}) DROP TABLE vecs_{element_type} diff --git a/tests/queries/0_stateless/02708_dotProduct.reference b/tests/queries/0_stateless/02708_dotProduct.reference index 5cc9a9f0502..593071a3521 100644 --- a/tests/queries/0_stateless/02708_dotProduct.reference +++ b/tests/queries/0_stateless/02708_dotProduct.reference @@ -32,3 +32,10 @@ 32 32 32 +-- Tests that trigger special paths +0 61 +1 186 +0 62 +1 187 +0 62 +1 187 diff --git a/tests/queries/0_stateless/02708_dotProduct.sql b/tests/queries/0_stateless/02708_dotProduct.sql index 2035e23cf1d..ac94ecc28d3 100644 --- a/tests/queries/0_stateless/02708_dotProduct.sql +++ b/tests/queries/0_stateless/02708_dotProduct.sql @@ -45,3 +45,12 @@ SELECT '-- Aliases'; SELECT scalarProduct([1, 2, 3], [4, 5, 6]); SELECT scalarProduct((1, 2, 3), (4, 5, 6)); SELECT arrayDotProduct([1, 2, 3], [4, 5, 6]); -- actually no alias but the internal function for arrays + +SELECT '-- Tests that trigger special paths'; +DROP TABLE IF EXISTS tab; +CREATE TABLE tab(id UInt64, vec Array(Float32)) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab VALUES (0, [0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0]) (1, [5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]); +SELECT id, arrayDotProduct(vec, vec) FROM tab ORDER BY id; -- non-const / non-const +SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float32), vec) FROM tab ORDER BY id; -- const / non-const +SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float64), vec) FROM tab ORDER BY id; -- const / non-const +DROP TABLE tab; From 87f3c957c7dc3d16d9967723e30215e12a0b5dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 6 Mar 2024 21:16:22 +0100 Subject: [PATCH 052/332] Blind experiment --- base/base/itoa.h | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/base/base/itoa.h b/base/base/itoa.h index 513070c99d9..c450090d82f 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -349,16 +349,32 @@ namespace convert template -static inline char * writeUIntText(T x, char * p) +static inline char * writeUIntText(T _x, char * p) { - static_assert(is_unsigned_v); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbit-int-extension" + int len = digits10(_x); + static_assert(std::is_same_v || std::is_same_v); + using T_ = std::conditional_t, unsigned __int128, unsigned _BitInt(256)>; +#pragma clang diagnostic pop - int len = digits10(x); - auto * pp = p + len; - while (x >= 100) + T_ x; + T_ hundred(100ULL); + if constexpr (std::is_same_v) { - const auto i = x % 100; - x /= 100; + x = (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); + } + else + { + x = (T_(_x.items[T::_impl::little(3)]) << 192) + (T_(_x.items[T::_impl::little(2)]) << 128) + + (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); + } + + auto * pp = p + len; + while (x >= hundred) + { + const auto i = x % hundred; + x /= hundred; pp -= 2; outTwoDigits(pp, i); } From c192d0b12532060d14934e60164df7ce771d9399 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 6 Mar 2024 22:29:41 +0100 Subject: [PATCH 053/332] impl --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 83 +++++++++++++------ src/Storages/MergeTree/IMergeTreeDataPart.h | 8 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 3 +- .../MergeTree/MergeTreeDataPartCompact.h | 12 +-- .../MergeTree/MergeTreeDataPartWide.cpp | 3 +- .../MergeTree/MergeTreeDataPartWide.h | 4 +- .../MergeTree/MergeTreeIndexGranularity.cpp | 5 +- .../MergeTree/MergeTreeIndexGranularity.h | 2 + .../MergeTreeIndexGranularityInfo.cpp | 14 +++- .../MergeTree/MergeTreeIndexGranularityInfo.h | 4 + 10 files changed, 97 insertions(+), 41 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a9bdceacef0..3922d5018c6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -2,38 +2,41 @@ #include #include +#include #include -#include #include +#include +#include #include -#include +#include +#include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include @@ -1966,7 +1969,39 @@ void IMergeTreeDataPart::checkConsistencyBase() const } } -void IMergeTreeDataPart::checkConsistency(bool /* require_part_metadata */) const +void IMergeTreeDataPart::checkConsistency(bool require_part_metadata) const +{ + try + { + checkConsistencyBase(); + doCheckConsistency(require_part_metadata); + } + catch (Exception & e) + { + const auto part_state = fmt::format( + "state: {}, is_unexpected_local_part: {}, is_frozen: {}, is_duplicate: {}", + stateString(), + is_unexpected_local_part, + is_frozen, + is_duplicate, + is_temp); + + const auto debug_info = fmt::format( + "columns: {}, getMarkSizeInBytes: {}, getMarksCount: {}, index_granularity_info: [{}], index_granularity: [{}], " + "part_state: [{}]", + columns.toString(), + index_granularity_info.getMarkSizeInBytes(columns.size()), + index_granularity.getMarksCount(), + index_granularity_info.describe(), + index_granularity.describe(), + part_state); + + e.addMessage(debug_info); + e.rethrow(); + } +} + +void IMergeTreeDataPart::doCheckConsistency(bool /* require_part_metadata */) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'checkConsistency' is not implemented for part with type {}", getType().toString()); } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 91c559d30c8..209c2d9a256 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -498,7 +498,7 @@ public: void writeChecksums(const MergeTreeDataPartChecksums & checksums_, const WriteSettings & settings); /// Checks the consistency of this data part. - virtual void checkConsistency(bool require_part_metadata) const; + void checkConsistency(bool require_part_metadata) const; /// Checks the consistency of this data part, and check the consistency of its projections (if any) as well. void checkConsistencyWithProjections(bool require_part_metadata) const; @@ -586,8 +586,6 @@ protected: void removeIfNeeded(); - void checkConsistencyBase() const; - /// Fill each_columns_size and total_size with sizes from columns files on /// disk using columns and checksums. virtual void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const = 0; @@ -610,6 +608,8 @@ protected: void initializeIndexGranularityInfo(); + virtual void doCheckConsistency(bool require_part_metadata) const; + private: String mutable_name; mutable MergeTreeDataPartState state{MergeTreeDataPartState::Temporary}; @@ -697,6 +697,8 @@ private: void incrementStateMetric(MergeTreeDataPartState state) const; void decrementStateMetric(MergeTreeDataPartState state) const; + void checkConsistencyBase() const; + /// This ugly flag is needed for debug assertions only mutable bool part_is_probably_removed_from_disk = false; }; diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 0ecd7abe183..5d4b602b5b8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -157,9 +157,8 @@ std::optional MergeTreeDataPartCompact::getColumnModificationTime(const return getDataPartStorage().getFileLastModified(DATA_FILE_NAME_WITH_EXTENSION).epochTime(); } -void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) const +void MergeTreeDataPartCompact::doCheckConsistency(bool require_part_metadata) const { - checkConsistencyBase(); String mrk_file_name = DATA_FILE_NAME + getMarksFileExtension(); if (!checksums.empty()) diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 35a358b3720..f897bcb0bfd 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -67,14 +67,14 @@ protected: MergeTreeIndexGranularity & index_granularity_, const MergeTreeIndexGranularityInfo & index_granularity_info_, size_t columns_count, const IDataPartStorage & data_part_storage_); -private: - void checkConsistency(bool require_part_metadata) const override; + void doCheckConsistency(bool require_part_metadata) const override; - /// Loads marks index granularity into memory - void loadIndexGranularity() override; + private: + /// Loads marks index granularity into memory + void loadIndexGranularity() override; - /// Compact parts doesn't support per column size, only total size - void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const override; + /// Compact parts doesn't support per column size, only total size + void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const override; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index dc6c1f0019d..0111f1e7b40 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -182,9 +182,8 @@ MergeTreeDataPartWide::~MergeTreeDataPartWide() removeIfNeeded(); } -void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const +void MergeTreeDataPartWide::doCheckConsistency(bool require_part_metadata) const { - checkConsistencyBase(); std::string marks_file_extension = index_granularity_info.mark_type.getFileExtension(); if (!checksums.empty()) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 14147c4ad56..508ea16d2d4 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -62,9 +62,9 @@ protected: MergeTreeIndexGranularity & index_granularity_, MergeTreeIndexGranularityInfo & index_granularity_info_, const IDataPartStorage & data_part_storage_, const std::string & any_column_file_name); -private: - void checkConsistency(bool require_part_metadata) const override; + void doCheckConsistency(bool require_part_metadata) const override; +private: /// Loads marks index granularity into memory void loadIndexGranularity() override; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp index 5fdd0555777..2a45ab1d927 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp @@ -121,5 +121,8 @@ void MergeTreeIndexGranularity::resizeWithFixedGranularity(size_t size, size_t f } } - +std::string MergeTreeIndexGranularity::describe() const +{ + return fmt::format("initialized: {}, marks_rows_partial_sums: [{}]", initialized, fmt::join(marks_rows_partial_sums, ", ")); +} } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/src/Storages/MergeTree/MergeTreeIndexGranularity.h index f5677995ae0..d67762f7293 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -95,6 +95,8 @@ public: /// Add `size` of marks with `fixed_granularity` rows void resizeWithFixedGranularity(size_t size, size_t fixed_granularity); + + std::string describe() const; }; } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index da89d52a9ff..1ff72a4e36d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -88,6 +88,10 @@ std::string MarkType::getFileExtension() const } } +std::string MarkType::describe() const +{ + return fmt::format("adaptive: {}, compressed: {}, part_type: {}", adaptive, compressed, part_type); +} std::optional MergeTreeIndexGranularityInfo::getMarksTypeFromFilesystem(const IDataPartStorage & data_part_storage) { @@ -132,10 +136,18 @@ size_t MergeTreeIndexGranularityInfo::getMarkSizeInBytes(size_t columns_num) con throw Exception(ErrorCodes::UNKNOWN_PART_TYPE, "Unknown part type"); } +std::string MergeTreeIndexGranularityInfo::describe() const +{ + return fmt::format( + "mark_type: [{}], index_granularity_bytes: {}, fixed_index_granularity: {}", + mark_type.describe(), + index_granularity_bytes, + fixed_index_granularity); +} + size_t getAdaptiveMrkSizeCompact(size_t columns_num) { /// Each mark contains number of rows in granule and two offsets for every column. return sizeof(UInt64) * (columns_num * 2 + 1); } - } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index af008866919..85006c3ffde 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -24,6 +24,8 @@ struct MarkType static bool isMarkFileExtension(std::string_view extension); std::string getFileExtension() const; + std::string describe() const; + bool adaptive = false; bool compressed = false; MergeTreeDataPartType::Value part_type = MergeTreeDataPartType::Unknown; @@ -58,6 +60,8 @@ public: size_t getMarkSizeInBytes(size_t columns_num = 1) const; static std::optional getMarksTypeFromFilesystem(const IDataPartStorage & data_part_storage); + + std::string describe() const; }; constexpr inline auto getNonAdaptiveMrkSizeWide() { return sizeof(UInt64) * 2; } From 2b52583e06056e19df97216f41b81102bca8bd9d Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 6 Mar 2024 23:01:42 +0100 Subject: [PATCH 054/332] fix style --- src/Storages/MergeTree/MergeTreeDataPartCompact.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index f897bcb0bfd..8bbec2808d7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -69,7 +69,7 @@ protected: void doCheckConsistency(bool require_part_metadata) const override; - private: +private: /// Loads marks index granularity into memory void loadIndexGranularity() override; From fb17749b50ce1024ef8c0b6f7bb8b7a58321894c Mon Sep 17 00:00:00 2001 From: Nikolay Monkov Date: Thu, 7 Mar 2024 09:45:24 +0500 Subject: [PATCH 055/332] file has been reformatted to pass Style check --- tests/ci/docker_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 803dbfcd92a..35c86d8eadd 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -216,7 +216,9 @@ def gen_tags(version: ClickHouseVersion, release_type: str) -> List[str]: return tags -def buildx_args(urls: Dict[str, str], arch: str, direct_urls: List[str], version: str) -> List[str]: +def buildx_args( + urls: Dict[str, str], arch: str, direct_urls: List[str], version: str +) -> List[str]: args = [ f"--platform=linux/{arch}", f"--label=build-url={GITHUB_RUN_URL}", @@ -268,7 +270,9 @@ def build_and_push_image( urls = [url for url in direct_urls[arch] if ".deb" in url] else: urls = [url for url in direct_urls[arch] if ".tgz" in url] - cmd_args.extend(buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe)) + cmd_args.extend( + buildx_args(repo_urls, arch, direct_urls=urls, version=version.describe) + ) if not push: cmd_args.append(f"--tag={image.repo}:{arch_tag}") cmd_args.extend( From f0a8d8843de5dffae2e1d4476fb119ad34059340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 7 Mar 2024 00:10:06 +0100 Subject: [PATCH 056/332] Not x86_64 are lagging behind in features --- base/base/itoa.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/base/base/itoa.h b/base/base/itoa.h index c450090d82f..a36eecaf1e5 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -351,12 +351,20 @@ namespace convert template static inline char * writeUIntText(T _x, char * p) { -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wbit-int-extension" int len = digits10(_x); static_assert(std::is_same_v || std::is_same_v); - using T_ = std::conditional_t, unsigned __int128, unsigned _BitInt(256)>; + using T_ = std::conditional_t< + std::is_same_v, + unsigned __int128, +#if defined(__x86_64__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbit-int-extension" + unsigned _BitInt(256) #pragma clang diagnostic pop +#else + T +#endif + >; T_ x; T_ hundred(100ULL); @@ -366,8 +374,12 @@ static inline char * writeUIntText(T _x, char * p) } else { +#if defined(__x86_64__) x = (T_(_x.items[T::_impl::little(3)]) << 192) + (T_(_x.items[T::_impl::little(2)]) << 128) + (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); +#else + x = _x; +#endif } auto * pp = p + len; From 66dea5111298abd4301df55b5615d158105fe78f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:40:48 +0100 Subject: [PATCH 057/332] fix clang-tidy --- src/Functions/array/arrayDotProduct.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index f9a6687e028..8b7c85e05dd 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -322,18 +322,18 @@ private: const auto & offsets_y = array_y.getOffsets(); ColumnArray::Offset prev_offset = 0; - for (size_t row = 0; row < offsets_y.size(); ++row) + for (auto offset_y : offsets_y) { - if (offsets_x[0] != offsets_y[row] - prev_offset) [[unlikely]] + if (offsets_x[0] != offset_y - prev_offset) [[unlikely]] { throw Exception( ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arguments of function {} have different array sizes: {} and {}", getName(), offsets_x[0], - offsets_y[row] - prev_offset); + offset_y - prev_offset); } - prev_offset = offsets_y[row]; + prev_offset = offset_y; } auto col_res = ColumnVector::create(); From 930deee699be05398aac334ce9e025d084c68a30 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 7 Mar 2024 22:02:10 +0800 Subject: [PATCH 058/332] fix bugs --- src/Columns/ColumnArray.cpp | 63 ++++++++++++++++++------------------- src/Columns/ColumnArray.h | 2 +- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 44b17c89ae1..0214375122f 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -43,29 +43,34 @@ namespace ErrorCodes static constexpr size_t max_array_size_as_field = 1000000; -ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); - - if (!offsets_concrete) - throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); - - if (!offsets_concrete->empty() && data && !data->empty()) + if (check_offsets) { - Offset last_offset = offsets_concrete->getData().back(); + const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); - /// This will also prevent possible overflow in offset. - if (data->size() != last_offset) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", - data->size(), last_offset); + if (!offsets_concrete) + throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); + + if (!offsets_concrete->empty() && data && !data->empty()) + { + Offset last_offset = offsets_concrete->getData().back(); + + /// This will also prevent possible overflow in offset. + if (data->size() != last_offset) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", + data->size(), + last_offset); + } + + /** NOTE + * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). + * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. + */ } - - /** NOTE - * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). - * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. - */ } ColumnArray::ColumnArray(MutableColumnPtr && nested_column) @@ -425,20 +430,14 @@ void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, if (tuple_size != src_tuple_size) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - MutableColumns temporary_arrays(tuple_size); - Columns src_temporary_arrays(tuple_size); - for (size_t i = 0; i < tuple_size; ++i) - { - temporary_arrays[i] = ColumnArray::create(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable()); - src_temporary_arrays[i] = ColumnArray::create(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - assert_cast(*temporary_arrays[i]) - .insertManyFromImpl(assert_cast(*src_temporary_arrays[i]), position, length, false); - } - Columns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) - tuple_columns[i] = assert_cast(*temporary_arrays[i]).getDataPtr(); - + { + auto array_of_element = ColumnArray(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable(), false); + auto src_array_of_element = ColumnArray(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); + array_of_element.insertManyFromImpl(src_array_of_element, position, length, false); + tuple_columns[i] = array_of_element.getDataPtr(); + } getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); } @@ -448,12 +447,12 @@ void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t positio const ColumnNullable & src_nullable = assert_cast(src.getData()); /// Process nested column without updating array offsets - auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); /// Process null map column without updating array offsets - auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()); + auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 765f86ec552..8c4d103e7d0 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -21,7 +21,7 @@ private: friend class COWHelper, ColumnArray>; /** Create an array column with specified values and offsets. */ - ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); + ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets = true); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ explicit ColumnArray(MutableColumnPtr && nested_column); From cd9d9018e0db8139e48cb722e9e9685d2a212c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 7 Mar 2024 17:15:42 +0100 Subject: [PATCH 059/332] Be able to iterate --- base/base/CMakeLists.txt | 1 + base/base/itoa.cpp | 503 +++++++++++++++++++++++++++++++++++ base/base/itoa.h | 498 +++------------------------------- src/Functions/CMakeLists.txt | 1 + 4 files changed, 540 insertions(+), 463 deletions(-) create mode 100644 base/base/itoa.cpp diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 548ba01d86a..55d046767b8 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -19,6 +19,7 @@ set (SRCS getPageSize.cpp getThreadId.cpp int8_to_string.cpp + itoa.cpp JSON.cpp mremap.cpp phdr_cache.cpp diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp new file mode 100644 index 00000000000..9fefc9f0f07 --- /dev/null +++ b/base/base/itoa.cpp @@ -0,0 +1,503 @@ +// Based on https://github.com/amdn/itoa and combined with our optimizations +// +//=== itoa.h - Fast integer to ascii conversion --*- C++ -*-// +// +// The MIT License (MIT) +// Copyright (c) 2016 Arturo Martin-de-Nicolas +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include +#include +#include +#include + + +template +int digits10(T x) +{ + if (x < T(10ULL)) + return 1; + if (x < T(100ULL)) + return 2; + if constexpr (sizeof(T) == 1) + return 3; + else + { + if (x < T(1000ULL)) + return 3; + + if (x < T(1000000000000ULL)) + { + if (x < T(100000000ULL)) + { + if (x < T(1000000ULL)) + { + if (x < T(10000ULL)) + return 4; + else + return 5 + (x >= T(100000ULL)); + } + + return 7 + (x >= T(10000000ULL)); + } + + if (x < T(10000000000ULL)) + return 9 + (x >= T(1000000000ULL)); + + return 11 + (x >= T(100000000000ULL)); + } + + return 12 + digits10(x / T(1000000000000ULL)); + } +} + + +namespace +{ + +template +static constexpr T pow10(size_t x) +{ + return x ? 10 * pow10(x - 1) : 1; +} + +// Division by a power of 10 is implemented using a multiplicative inverse. +// This strength reduction is also done by optimizing compilers, but +// presently the fastest results are produced by using the values +// for the multiplication and the shift as given by the algorithm +// described by Agner Fog in "Optimizing Subroutines in Assembly Language" +// +// http://www.agner.org/optimize/optimizing_assembly.pdf +// +// "Integer division by a constant (all processors) +// A floating point number can be divided by a constant by multiplying +// with the reciprocal. If we want to do the same with integers, we have +// to scale the reciprocal by 2n and then shift the product to the right +// by n. There are various algorithms for finding a suitable value of n +// and compensating for rounding errors. The algorithm described below +// was invented by Terje Mathisen, Norway, and not published elsewhere." + +/// Division by constant is performed by: +/// 1. Adding 1 if needed; +/// 2. Multiplying by another constant; +/// 3. Shifting right by another constant. +template +struct Division +{ + static constexpr bool add{add_}; + static constexpr UInt multiplier{multiplier_}; + static constexpr unsigned shift{shift_}; +}; + +/// Select a type with appropriate number of bytes from the list of types. +/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes. +/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t. +template +struct SelectType +{ + using Result = typename SelectType::Result; +}; + +template +struct SelectType<1, T, Ts...> +{ + using Result = T; +}; + + +/// Division by 10^N where N is the size of the type. +template +using DivisionBy10PowN = typename SelectType< + N, + Division, /// divide by 10 + Division, /// divide by 100 + Division, /// divide by 10000 + Division /// divide by 100000000 + >::Result; + +template +using UnsignedOfSize = typename SelectType::Result; + +/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in +template +struct QuotientAndRemainder +{ + UnsignedOfSize quotient; // quotient with fewer than 2*N decimal digits + UnsignedOfSize remainder; // remainder with at most N decimal digits +}; + +template +QuotientAndRemainder static inline split(UnsignedOfSize value) +{ + constexpr DivisionBy10PowN division; + + UnsignedOfSize quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift; + UnsignedOfSize remainder = static_cast>(value - quotient * pow10>(N)); + + return {quotient, remainder}; +} + + +static inline char * outDigit(char * p, uint8_t value) +{ + *p = '0' + value; + ++p; + return p; +} + +// Using a lookup table to convert binary numbers from 0 to 99 +// into ascii characters as described by Andrei Alexandrescu in +// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/ + +static const char digits[201] = "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899"; + +static inline char * outTwoDigits(char * p, uint8_t value) +{ + memcpy(p, &digits[value * 2], 2); + p += 2; + return p; +} + + +namespace convert +{ +template +static char * head(char * p, UInt u); +template +static char * tail(char * p, UInt u); + +//===----------------------------------------------------------===// +// head: find most significant digit, skip leading zeros +//===----------------------------------------------------------===// + +// "x" contains quotient and remainder after division by 10^N +// quotient is less than 10^N +template +static inline char * head(char * p, QuotientAndRemainder x) +{ + p = head(p, UnsignedOfSize(x.quotient)); + p = tail(p, x.remainder); + return p; +} + +// "u" is less than 10^2*N +template +static inline char * head(char * p, UInt u) +{ + return u < pow10>(N) ? head(p, UnsignedOfSize(u)) : head(p, split(u)); +} + +// recursion base case, selected when "u" is one byte +template <> +inline char * head, 1>(char * p, UnsignedOfSize<1> u) +{ + return u < 10 ? outDigit(p, u) : outTwoDigits(p, u); +} + +//===----------------------------------------------------------===// +// tail: produce all digits including leading zeros +//===----------------------------------------------------------===// + +// recursive step, "u" is less than 10^2*N +template +static inline char * tail(char * p, UInt u) +{ + QuotientAndRemainder x = split(u); + p = tail(p, UnsignedOfSize(x.quotient)); + p = tail(p, x.remainder); + return p; +} + +// recursion base case, selected when "u" is one byte +template <> +inline char * tail, 1>(char * p, UnsignedOfSize<1> u) +{ + return outTwoDigits(p, u); +} + +//===----------------------------------------------------------===// +// large values are >= 10^2*N +// where x contains quotient and remainder after division by 10^N +//===----------------------------------------------------------===// + +template +static inline char * large(char * p, QuotientAndRemainder x) +{ + QuotientAndRemainder y = split(x.quotient); + p = head(p, UnsignedOfSize(y.quotient)); + p = tail(p, y.remainder); + p = tail(p, x.remainder); + return p; +} + +//===----------------------------------------------------------===// +// handle values of "u" that might be >= 10^2*N +// where N is the size of "u" in bytes +//===----------------------------------------------------------===// + +template +static inline char * uitoa(char * p, UInt u) +{ + if (u < pow10>(N)) + return head(p, UnsignedOfSize(u)); + QuotientAndRemainder x = split(u); + + return u < pow10>(2 * N) ? head(p, x) : large(p, x); +} + +// selected when "u" is one byte +template <> +inline char * uitoa, 1>(char * p, UnsignedOfSize<1> u) +{ + if (u < 10) + return outDigit(p, u); + else if (u < 100) + return outTwoDigits(p, u); + else + { + p = outDigit(p, u / 100); + p = outTwoDigits(p, u % 100); + return p; + } +} + +//===----------------------------------------------------------===// +// handle unsigned and signed integral operands +//===----------------------------------------------------------===// + +// itoa: handle unsigned integral operands (selected by SFINAE) +template && std::is_integral_v> * = nullptr> +static inline char * itoa(U u, char * p) +{ + return convert::uitoa(p, u); +} + +// itoa: handle signed integral operands (selected by SFINAE) +template && std::is_integral_v> * = nullptr> +static inline char * itoa(I i, char * p) +{ + // Need "mask" to be filled with a copy of the sign bit. + // If "i" is a negative value, then the result of "operator >>" + // is implementation-defined, though usually it is an arithmetic + // right shift that replicates the sign bit. + // Use a conditional expression to be portable, + // a good optimizing compiler generates an arithmetic right shift + // and avoids the conditional branch. + UnsignedOfSize mask = i < 0 ? ~UnsignedOfSize(0) : 0; + // Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize. + // Cannot use std::abs() because the result is undefined + // in 2's complement systems for the most-negative value. + // Want to avoid conditional branch for performance reasons since + // CPU branch prediction will be ineffective when negative values + // occur randomly. + // Let "u" be "i" cast to unsigned type UnsignedOfSize. + // Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative. + // This yields the absolute value with the desired type without + // using a conditional branch and without invoking undefined or + // implementation defined behavior: + UnsignedOfSize u = ((2 * UnsignedOfSize(i)) & ~mask) - UnsignedOfSize(i); + // Unconditionally store a minus sign when producing digits + // in a forward direction and increment the pointer only if + // the value is in fact negative. + // This avoids a conditional branch and is safe because we will + // always produce at least one digit and it will overwrite the + // minus sign when the value is not negative. + *p = '-'; + p += (mask & 1); + p = convert::uitoa(p, u); + return p; +} +} + + +template +static NO_INLINE char * writeUIntText(T _x, char * p) +{ + static_assert(std::is_same_v || std::is_same_v); + using T_ = std::conditional_t< + std::is_same_v, + unsigned __int128, +#if defined(__x86_64__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wbit-int-extension" + unsigned _BitInt(256) +# pragma clang diagnostic pop +#else + T +#endif + >; + + T_ x; + T_ hundred(100ULL); + if constexpr (std::is_same_v) + { + x = (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); + } + else + { +#if defined(__x86_64__) + x = (T_(_x.items[T::_impl::little(3)]) << 192) + (T_(_x.items[T::_impl::little(2)]) << 128) + + (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); +#else + x = _x; +#endif + } + + int len = digits10(x); + auto * pp = p + len; + while (x >= hundred) + { + const auto i = x % hundred; + x /= hundred; + pp -= 2; + outTwoDigits(pp, i); + } + if (x < 10) + *p = '0' + x; + else + outTwoDigits(p, x); + return p + len; +} + +static ALWAYS_INLINE inline char * writeLeadingMinus(char * pos) +{ + *pos = '-'; + return pos + 1; +} + +template +static ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) +{ + static_assert(std::is_same_v || std::is_same_v); + + using UnsignedT = make_unsigned_t; + static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1); + + if (unlikely(x == min_int)) + { + if constexpr (std::is_same_v) + { + const char * res = "-170141183460469231731687303715884105728"; + memcpy(pos, res, strlen(res)); + return pos + strlen(res); + } + else if constexpr (std::is_same_v) + { + const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968"; + memcpy(pos, res, strlen(res)); + return pos + strlen(res); + } + } + + if (x < 0) + { + x = -x; + pos = writeLeadingMinus(pos); + } + return writeUIntText(UnsignedT(x), pos); +} +} + +template +char * itoa(T i, char * p) +{ + return convert::itoa(i, p); +} + +template <> +char * itoa(UInt8 i, char * p) +{ + return convert::itoa(uint8_t(i), p); +} + +template <> +char * itoa(Int8 i, char * p) +{ + return convert::itoa(int8_t(i), p); +} + +template <> +char * itoa(UInt128 i, char * p) +{ + return writeUIntText(i, p); +} + +template <> +char * itoa(Int128 i, char * p) +{ + return writeSIntText(i, p); +} + +template <> +char * itoa(UInt256 i, char * p) +{ + return writeUIntText(i, p); +} + +template <> +char * itoa(Int256 i, char * p) +{ + return writeSIntText(i, p); +} + +#define FOR_MISSING_INTEGER_TYPES(M) \ + M(int8_t) \ + M(uint8_t) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(Int16) \ + M(Int32) \ + M(Int64) + +#define INSTANTIATION(T) template char * itoa(T i, char * p); +FOR_MISSING_INTEGER_TYPES(INSTANTIATION) + +#undef FOR_MISSING_INTEGER_TYPES +#undef INSTANTIATION + + +#define DIGITS_INTEGER_TYPES(M) \ + M(uint8_t) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(UInt128) \ + M(UInt256) + +#define INSTANTIATION(T) template int digits10(T x); +DIGITS_INTEGER_TYPES(INSTANTIATION) + +#undef DIGITS_INTEGER_TYPES +#undef INSTANTIATION diff --git a/base/base/itoa.h b/base/base/itoa.h index a36eecaf1e5..71603cdeb88 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -1,474 +1,46 @@ #pragma once -// Based on https://github.com/amdn/itoa and combined with our optimizations -// -//=== itoa.h - Fast integer to ascii conversion --*- C++ -*-// -// -// The MIT License (MIT) -// Copyright (c) 2016 Arturo Martin-de-Nicolas -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -//===----------------------------------------------------------------------===// - -#include -#include -#include -#include #include +template char * itoa(T i, char * p); -template -inline int digits10(T x) -{ - if (x < 10ULL) - return 1; - if (x < 100ULL) - return 2; - if (x < 1000ULL) - return 3; +template <> char * itoa(UInt8 i, char * p); +template <> char * itoa(Int8 i, char * p); +template <> char * itoa(UInt128 i, char * p); +template <> char * itoa(Int128 i, char * p); +template <> char * itoa(UInt256 i, char * p); +template <> char * itoa(Int256 i, char * p); - if (x < 1000000000000ULL) - { - if (x < 100000000ULL) - { - if (x < 1000000ULL) - { - if (x < 10000ULL) - return 4; - else - return 5 + (x >= 100000ULL); - } +#define FOR_MISSING_INTEGER_TYPES(M) \ + M(int8_t) \ + M(uint8_t) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(Int16) \ + M(Int32) \ + M(Int64) - return 7 + (x >= 10000000ULL); - } +#define INSTANTIATION(T) \ + extern template char * itoa(T i, char * p); +FOR_MISSING_INTEGER_TYPES(INSTANTIATION) - if (x < 10000000000ULL) - return 9 + (x >= 1000000000ULL); - - return 11 + (x >= 100000000000ULL); - } - - return 12 + digits10(x / 1000000000000ULL); -} +#undef FOR_MISSING_INTEGER_TYPES +#undef INSTANTIATION -namespace impl -{ +template int digits10(T x); -template -static constexpr T pow10(size_t x) -{ - return x ? 10 * pow10(x - 1) : 1; -} - -// Division by a power of 10 is implemented using a multiplicative inverse. -// This strength reduction is also done by optimizing compilers, but -// presently the fastest results are produced by using the values -// for the multiplication and the shift as given by the algorithm -// described by Agner Fog in "Optimizing Subroutines in Assembly Language" -// -// http://www.agner.org/optimize/optimizing_assembly.pdf -// -// "Integer division by a constant (all processors) -// A floating point number can be divided by a constant by multiplying -// with the reciprocal. If we want to do the same with integers, we have -// to scale the reciprocal by 2n and then shift the product to the right -// by n. There are various algorithms for finding a suitable value of n -// and compensating for rounding errors. The algorithm described below -// was invented by Terje Mathisen, Norway, and not published elsewhere." - -/// Division by constant is performed by: -/// 1. Adding 1 if needed; -/// 2. Multiplying by another constant; -/// 3. Shifting right by another constant. -template -struct Division -{ - static constexpr bool add{add_}; - static constexpr UInt multiplier{multiplier_}; - static constexpr unsigned shift{shift_}; -}; - -/// Select a type with appropriate number of bytes from the list of types. -/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes. -/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t. -template -struct SelectType -{ - using Result = typename SelectType::Result; -}; - -template -struct SelectType<1, T, Ts...> -{ - using Result = T; -}; - - -/// Division by 10^N where N is the size of the type. -template -using DivisionBy10PowN = typename SelectType -< - N, - Division, /// divide by 10 - Division, /// divide by 100 - Division, /// divide by 10000 - Division /// divide by 100000000 ->::Result; - -template -using UnsignedOfSize = typename SelectType -< - N, - uint8_t, - uint16_t, - uint32_t, - uint64_t, - __uint128_t ->::Result; - -/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in -template -struct QuotientAndRemainder -{ - UnsignedOfSize quotient; // quotient with fewer than 2*N decimal digits - UnsignedOfSize remainder; // remainder with at most N decimal digits -}; - -template -QuotientAndRemainder static inline split(UnsignedOfSize value) -{ - constexpr DivisionBy10PowN division; - - UnsignedOfSize quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift; - UnsignedOfSize remainder = static_cast>(value - quotient * pow10>(N)); - - return {quotient, remainder}; -} - - -static inline char * outDigit(char * p, uint8_t value) -{ - *p = '0' + value; - ++p; - return p; -} - -// Using a lookup table to convert binary numbers from 0 to 99 -// into ascii characters as described by Andrei Alexandrescu in -// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/ - -static const char digits[201] = "00010203040506070809" - "10111213141516171819" - "20212223242526272829" - "30313233343536373839" - "40414243444546474849" - "50515253545556575859" - "60616263646566676869" - "70717273747576777879" - "80818283848586878889" - "90919293949596979899"; - -static inline char * outTwoDigits(char * p, uint8_t value) -{ - memcpy(p, &digits[value * 2], 2); - p += 2; - return p; -} - - -namespace convert -{ - template static char * head(char * p, UInt u); - template static char * tail(char * p, UInt u); - - //===----------------------------------------------------------===// - // head: find most significant digit, skip leading zeros - //===----------------------------------------------------------===// - - // "x" contains quotient and remainder after division by 10^N - // quotient is less than 10^N - template - static inline char * head(char * p, QuotientAndRemainder x) - { - p = head(p, UnsignedOfSize(x.quotient)); - p = tail(p, x.remainder); - return p; - } - - // "u" is less than 10^2*N - template - static inline char * head(char * p, UInt u) - { - return u < pow10>(N) - ? head(p, UnsignedOfSize(u)) - : head(p, split(u)); - } - - // recursion base case, selected when "u" is one byte - template <> - inline char * head, 1>(char * p, UnsignedOfSize<1> u) - { - return u < 10 - ? outDigit(p, u) - : outTwoDigits(p, u); - } - - //===----------------------------------------------------------===// - // tail: produce all digits including leading zeros - //===----------------------------------------------------------===// - - // recursive step, "u" is less than 10^2*N - template - static inline char * tail(char * p, UInt u) - { - QuotientAndRemainder x = split(u); - p = tail(p, UnsignedOfSize(x.quotient)); - p = tail(p, x.remainder); - return p; - } - - // recursion base case, selected when "u" is one byte - template <> - inline char * tail, 1>(char * p, UnsignedOfSize<1> u) - { - return outTwoDigits(p, u); - } - - //===----------------------------------------------------------===// - // large values are >= 10^2*N - // where x contains quotient and remainder after division by 10^N - //===----------------------------------------------------------===// - - template - static inline char * large(char * p, QuotientAndRemainder x) - { - QuotientAndRemainder y = split(x.quotient); - p = head(p, UnsignedOfSize(y.quotient)); - p = tail(p, y.remainder); - p = tail(p, x.remainder); - return p; - } - - //===----------------------------------------------------------===// - // handle values of "u" that might be >= 10^2*N - // where N is the size of "u" in bytes - //===----------------------------------------------------------===// - - template - static inline char * uitoa(char * p, UInt u) - { - if (u < pow10>(N)) - return head(p, UnsignedOfSize(u)); - QuotientAndRemainder x = split(u); - - return u < pow10>(2 * N) - ? head(p, x) - : large(p, x); - } - - // selected when "u" is one byte - template <> - inline char * uitoa, 1>(char * p, UnsignedOfSize<1> u) - { - if (u < 10) - return outDigit(p, u); - else if (u < 100) - return outTwoDigits(p, u); - else - { - p = outDigit(p, u / 100); - p = outTwoDigits(p, u % 100); - return p; - } - } - - //===----------------------------------------------------------===// - // handle unsigned and signed integral operands - //===----------------------------------------------------------===// - - // itoa: handle unsigned integral operands (selected by SFINAE) - template && std::is_integral_v> * = nullptr> - static inline char * itoa(U u, char * p) - { - return convert::uitoa(p, u); - } - - // itoa: handle signed integral operands (selected by SFINAE) - template && std::is_integral_v> * = nullptr> - static inline char * itoa(I i, char * p) - { - // Need "mask" to be filled with a copy of the sign bit. - // If "i" is a negative value, then the result of "operator >>" - // is implementation-defined, though usually it is an arithmetic - // right shift that replicates the sign bit. - // Use a conditional expression to be portable, - // a good optimizing compiler generates an arithmetic right shift - // and avoids the conditional branch. - UnsignedOfSize mask = i < 0 ? ~UnsignedOfSize(0) : 0; - // Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize. - // Cannot use std::abs() because the result is undefined - // in 2's complement systems for the most-negative value. - // Want to avoid conditional branch for performance reasons since - // CPU branch prediction will be ineffective when negative values - // occur randomly. - // Let "u" be "i" cast to unsigned type UnsignedOfSize. - // Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative. - // This yields the absolute value with the desired type without - // using a conditional branch and without invoking undefined or - // implementation defined behavior: - UnsignedOfSize u = ((2 * UnsignedOfSize(i)) & ~mask) - UnsignedOfSize(i); - // Unconditionally store a minus sign when producing digits - // in a forward direction and increment the pointer only if - // the value is in fact negative. - // This avoids a conditional branch and is safe because we will - // always produce at least one digit and it will overwrite the - // minus sign when the value is not negative. - *p = '-'; - p += (mask & 1); - p = convert::uitoa(p, u); - return p; - } -} - - -template -static inline char * writeUIntText(T _x, char * p) -{ - int len = digits10(_x); - static_assert(std::is_same_v || std::is_same_v); - using T_ = std::conditional_t< - std::is_same_v, - unsigned __int128, -#if defined(__x86_64__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wbit-int-extension" - unsigned _BitInt(256) -#pragma clang diagnostic pop -#else - T -#endif - >; - - T_ x; - T_ hundred(100ULL); - if constexpr (std::is_same_v) - { - x = (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); - } - else - { -#if defined(__x86_64__) - x = (T_(_x.items[T::_impl::little(3)]) << 192) + (T_(_x.items[T::_impl::little(2)]) << 128) + - (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); -#else - x = _x; -#endif - } - - auto * pp = p + len; - while (x >= hundred) - { - const auto i = x % hundred; - x /= hundred; - pp -= 2; - outTwoDigits(pp, i); - } - if (x < 10) - *p = '0' + x; - else - outTwoDigits(p, x); - return p + len; -} - -static inline char * writeLeadingMinus(char * pos) -{ - *pos = '-'; - return pos + 1; -} - -template -static inline char * writeSIntText(T x, char * pos) -{ - static_assert(std::is_same_v || std::is_same_v); - - using UnsignedT = make_unsigned_t; - static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1); - - if (unlikely(x == min_int)) - { - if constexpr (std::is_same_v) - { - const char * res = "-170141183460469231731687303715884105728"; - memcpy(pos, res, strlen(res)); - return pos + strlen(res); - } - else if constexpr (std::is_same_v) - { - const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968"; - memcpy(pos, res, strlen(res)); - return pos + strlen(res); - } - } - - if (x < 0) - { - x = -x; - pos = writeLeadingMinus(pos); - } - return writeUIntText(UnsignedT(x), pos); -} - -} - -template -char * itoa(I i, char * p) -{ - return impl::convert::itoa(i, p); -} - -template <> -inline char * itoa(char8_t i, char * p) -{ - return impl::convert::itoa(uint8_t(i), p); -} - -template <> -inline char * itoa(UInt128 i, char * p) -{ - return impl::writeUIntText(i, p); -} - -template <> -inline char * itoa(Int128 i, char * p) -{ - return impl::writeSIntText(i, p); -} - -template <> -inline char * itoa(UInt256 i, char * p) -{ - return impl::writeUIntText(i, p); -} - -template <> -inline char * itoa(Int256 i, char * p) -{ - return impl::writeSIntText(i, p); -} +#define DIGITS_INTEGER_TYPES(M) \ + M(uint8_t) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(UInt128) \ + M(UInt256) +#define INSTANTIATION(T) \ + extern template int digits10(T x); +DIGITS_INTEGER_TYPES(INSTANTIATION) +#undef DIGITS_INTEGER_TYPES +#undef INSTANTIATION diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index ac3e3671ae0..dea369a508a 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -37,6 +37,7 @@ list (APPEND PUBLIC_LIBS clickhouse_dictionaries_embedded clickhouse_parsers ch_contrib::consistent_hashing + common dbms ch_contrib::metrohash ch_contrib::murmurhash From 0f0ea422f21af8e37aa5c8ef58002d608cde5c77 Mon Sep 17 00:00:00 2001 From: serxa Date: Thu, 7 Mar 2024 17:05:54 +0000 Subject: [PATCH 060/332] separate limits on number of waiting and executing queries --- .../settings.md | 16 ++- programs/server/Server.cpp | 1 + src/Common/AsyncLoader.cpp | 21 +++- src/Common/AsyncLoader.h | 65 ++++++++++- src/Common/tests/gtest_async_loader.cpp | 66 +++++++++++ src/Core/ServerSettings.h | 1 + src/Interpreters/ProcessList.cpp | 103 +++++++++++++++--- src/Interpreters/ProcessList.h | 41 ++++++- .../System/StorageSystemServerSettings.cpp | 1 + 9 files changed, 293 insertions(+), 22 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 07c9a2b88ab..63fbd9d1964 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -379,6 +379,18 @@ Type: UInt64 Default: 0 +## max_waiting_queries + +Limit on total number of concurrently waiting queries. Execution of a waiting query is blocked while required tables are loading asynchronously (see `async_load_databases`). Note that waiting queries are not counted when `max_concurrent_queries`, `max_concurrent_insert_queries`, `max_concurrent_select_queries`, `max_concurrent_queries_for_user` and `max_concurrent_queries_for_all_users` limits are checked. This correction is done to avoid hitting these limits just after server startup. Zero means unlimited. + +:::note +This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged. +::: + +Type: UInt64 + +Default: 0 + ## max_connections Max server connections. @@ -1725,7 +1737,7 @@ Default value: `0.5`. Asynchronous loading of databases and tables. -If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. +If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. Also consider setting a limit `max_waiting_queries` for the total number of waiting queries. If `false`, all databases are loaded when the server starts. @@ -2926,7 +2938,7 @@ Default: 0 ## ignore_empty_sql_security_in_create_view_query {#ignore_empty_sql_security_in_create_view_query} -If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. +If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. :::note This setting is only necessary for the migration period and will become obsolete in 24.4 diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a10f47be0b8..336563665a2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1429,6 +1429,7 @@ try global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries); global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries); global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries); + global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries); if (config->has("keeper_server")) global_context->updateKeeperConfiguration(*config); diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 140194e10b4..80e4c72f1c1 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -140,6 +140,11 @@ void LoadJob::finish() finish_time = std::chrono::system_clock::now(); if (waiters > 0) finished.notify_all(); + else + { + on_waiters_increment = {}; + on_waiters_decrement = {}; + } } void LoadJob::scheduled(UInt64 job_id_) @@ -765,11 +770,25 @@ void AsyncLoader::wait(std::unique_lock & job_lock, const LoadJobPtr if (job->load_status != LoadStatus::PENDING) // Shortcut just to avoid incrementing ProfileEvents return; + if (job->on_waiters_increment) + job->on_waiters_increment(job); + + // WARNING: it is important not to throw below this point to avoid `on_waiters_increment` call w/o matching `on_waiters_decrement` call + Stopwatch watch; job->waiters++; job->finished.wait(job_lock, [&] { return job->load_status != LoadStatus::PENDING; }); job->waiters--; ProfileEvents::increment(ProfileEvents::AsyncLoaderWaitMicroseconds, watch.elapsedMicroseconds()); + + if (job->on_waiters_decrement) + job->on_waiters_decrement(job); + + if (job->waiters == 0) + { + job->on_waiters_increment = {}; + job->on_waiters_decrement = {}; + } } bool AsyncLoader::canSpawnWorker(Pool & pool, std::unique_lock &) @@ -859,7 +878,7 @@ void AsyncLoader::worker(Pool & pool) try { current_load_job = job.get(); - SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported + SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported, but jobs can wait one another job->execute(*this, pool_id, job); exception_from_job = {}; } diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index b1b336d24dc..3f81a36aa96 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -59,7 +59,8 @@ enum class LoadStatus class LoadJob : private boost::noncopyable { public: - template + // NOTE: makeLoadJob() helper should be used instead of direct ctor call + template LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, DFFunc && dependency_failure_, Func && func_) : dependencies(std::forward(dependencies_)) , name(std::move(name_)) @@ -69,6 +70,19 @@ public: , func(std::forward(func_)) {} + // NOTE: makeLoadJob() helper should be used instead of direct ctor call + template + LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, WIFunc && on_waiters_increment_, WDFunc && on_waiters_decrement_, DFFunc && dependency_failure_, Func && func_) + : dependencies(std::forward(dependencies_)) + , name(std::move(name_)) + , execution_pool_id(pool_id_) + , pool_id(pool_id_) + , on_waiters_increment(std::forward(on_waiters_increment_)) + , on_waiters_decrement(std::forward(on_waiters_decrement_)) + , dependency_failure(std::forward(dependency_failure_)) + , func(std::forward(func_)) + {} + // Current job status. LoadStatus status() const; std::exception_ptr exception() const; @@ -112,6 +126,13 @@ private: std::atomic execution_pool_id; std::atomic pool_id; + // Handlers that is called by every new waiting thread, just before going to sleep. + // If `on_waiters_increment` throws, then wait is canceled, and corresponding `on_waiters_decrement` will never be called. + // It can be used for counting and limits on number of waiters. + // Note that implementations are called under `LoadJob::mutex` and should be fast. + std::function on_waiters_increment; + std::function on_waiters_decrement; + // Handler for failed or canceled dependencies. // If job needs to be canceled on `dependency` failure, then function should set `cancel` to a specific reason. // Note that implementation should be fast and cannot use AsyncLoader, because it is called under `AsyncLoader::mutex`. @@ -140,8 +161,50 @@ void cancelOnDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & depen void ignoreDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & dependency, std::exception_ptr & cancel); template concept LoadJobDependencyFailure = std::invocable; +template concept LoadJobOnWaiters = std::invocable; template concept LoadJobFunc = std::invocable; +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward(dependency_failure), std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + +LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func) +{ + return std::make_shared(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward(func)); +} + + LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func) { return std::make_shared(std::move(dependencies), std::move(name), 0, std::forward(dependency_failure), std::forward(func)); diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp index fc2537abcfc..62a27f259cc 100644 --- a/src/Common/tests/gtest_async_loader.cpp +++ b/src/Common/tests/gtest_async_loader.cpp @@ -643,6 +643,72 @@ TEST(AsyncLoader, CustomDependencyFailure) ASSERT_EQ(good_count.load(), 3); } +TEST(AsyncLoader, WaitersLimit) +{ + AsyncLoaderTest t(16); + + std::atomic waiters_total{0}; + int waiters_limit = 5; + auto waiters_inc = [&] (const LoadJobPtr &) { + int value = waiters_total.load(); + while (true) + { + if (value >= waiters_limit) + throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "Too many waiters: {}", value); + if (waiters_total.compare_exchange_strong(value, value + 1)) + break; + } + }; + auto waiters_dec = [&] (const LoadJobPtr &) { + waiters_total.fetch_sub(1); + }; + + std::barrier sync(2); + t.loader.start(); + + auto job_func = [&] (AsyncLoader &, const LoadJobPtr &) { + sync.arrive_and_wait(); // (A) + }; + + auto job = makeLoadJob({}, "job", waiters_inc, waiters_dec, job_func); + auto task = t.schedule({job}); + + std::atomic failure{0}; + std::atomic success{0}; + std::vector waiters; + waiters.reserve(10); + auto waiter = [&] { + try + { + t.loader.wait(job); + success.fetch_add(1); + } + catch(...) + { + failure.fetch_add(1); + } + }; + + for (int i = 0; i < 10; i++) + waiters.emplace_back(waiter); + + while (failure.load() != 5) + std::this_thread::yield(); + + ASSERT_EQ(job->waitersCount(), 5); + + sync.arrive_and_wait(); // (A) + + for (auto & thread : waiters) + thread.join(); + + ASSERT_EQ(success.load(), 5); + ASSERT_EQ(failure.load(), 5); + ASSERT_EQ(waiters_total.load(), 0); + + t.loader.wait(); +} + TEST(AsyncLoader, TestConcurrency) { AsyncLoaderTest t(10); diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index c82255ec59c..129b1016fca 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -64,6 +64,7 @@ namespace DB M(UInt64, max_concurrent_queries, 0, "Maximum number of concurrently executed queries. Zero means unlimited.", 0) \ M(UInt64, max_concurrent_insert_queries, 0, "Maximum number of concurrently INSERT queries. Zero means unlimited.", 0) \ M(UInt64, max_concurrent_select_queries, 0, "Maximum number of concurrently SELECT queries. Zero means unlimited.", 0) \ + M(UInt64, max_waiting_queries, 0, "Maximum number of concurrently waiting queries blocked due to `async_load_databases`. Note that waiting queries are not considered by `max_concurrent_*queries*` limits. Zero means unlimited.", 0) \ \ M(Double, cache_size_to_ram_max_ratio, 0.5, "Set cache size ro RAM max ratio. Allows to lower cache size on low-memory systems.", 0) \ M(String, uncompressed_cache_policy, DEFAULT_UNCOMPRESSED_CACHE_POLICY, "Uncompressed cache policy name.", 0) \ diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3bd7b2d4206..f451d561e60 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -83,25 +83,31 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q IAST::QueryKind query_kind = ast->getQueryKind(); const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); - if (!is_unlimited_query && max_size && processes.size() >= max_size) + UInt64 waiting_queries = waiting_queries_amount.load(); + if (!is_unlimited_query && max_size && processes.size() >= max_size + waiting_queries) { if (queue_max_wait_ms) LOG_WARNING(getLogger("ProcessList"), "Too many simultaneous queries, will wait {} ms.", queue_max_wait_ms); - if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), [&]{ return processes.size() < max_size; })) - throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries. Maximum: {}", max_size); + if (!queue_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(queue_max_wait_ms), + [&]{ waiting_queries = waiting_queries_amount.load(); return processes.size() < max_size + waiting_queries; })) + throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous queries. Maximum: {}{}", + max_size, waiting_queries == 0 ? "" : fmt::format(", waiting: {}", waiting_queries)); } if (!is_unlimited_query) { QueryAmount amount = getQueryKindAmount(query_kind); - if (max_insert_queries_amount && query_kind == IAST::QueryKind::Insert && amount >= max_insert_queries_amount) + UInt64 waiting_inserts = waiting_insert_queries_amount.load(); + UInt64 waiting_selects = waiting_select_queries_amount.load(); + if (max_insert_queries_amount && query_kind == IAST::QueryKind::Insert && amount >= max_insert_queries_amount + waiting_inserts) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, - "Too many simultaneous insert queries. Maximum: {}, current: {}", - max_insert_queries_amount, amount); - if (max_select_queries_amount && query_kind == IAST::QueryKind::Select && amount >= max_select_queries_amount) + "Too many simultaneous insert queries. Maximum: {}, current: {}{}", + max_insert_queries_amount, amount, waiting_inserts == 0 ? "" : fmt::format(", waiting: {}", waiting_inserts)); + if (max_select_queries_amount && query_kind == IAST::QueryKind::Select && amount >= max_select_queries_amount + waiting_selects) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, - "Too many simultaneous select queries. Maximum: {}, current: {}", - max_select_queries_amount, amount); + "Too many simultaneous select queries. Maximum: {}, current: {}{}", + max_select_queries_amount, amount, waiting_selects == 0 ? "" : fmt::format(", waiting: {}", waiting_selects)); } { @@ -124,10 +130,12 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q * once is already processing 50+ concurrent queries (including analysts or any other users). */ + waiting_queries = waiting_queries_amount.load(); if (!is_unlimited_query && settings.max_concurrent_queries_for_all_users - && processes.size() >= settings.max_concurrent_queries_for_all_users) + && processes.size() >= settings.max_concurrent_queries_for_all_users + waiting_queries_amount) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for all users. " - "Current: {}, maximum: {}", processes.size(), settings.max_concurrent_queries_for_all_users.toString()); + "Current: {}, maximum: {}{}", processes.size(), settings.max_concurrent_queries_for_all_users.toString(), + waiting_queries == 0 ? "" : fmt::format(", waiting: {}", waiting_queries)); } /** Why we use current user? @@ -145,13 +153,15 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q if (user_process_list != user_to_queries.end()) { + UInt64 user_waiting_queries = user_process_list->second.waiting_queries_amount.load(); if (!is_unlimited_query && settings.max_concurrent_queries_for_user - && user_process_list->second.queries.size() >= settings.max_concurrent_queries_for_user) + && user_process_list->second.queries.size() >= settings.max_concurrent_queries_for_user + user_waiting_queries) throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for user {}. " - "Current: {}, maximum: {}", + "Current: {}, maximum: {}{}", client_info.current_user, user_process_list->second.queries.size(), - settings.max_concurrent_queries_for_user.toString()); + settings.max_concurrent_queries_for_user.toString(), + user_waiting_queries == 0 ? "" : fmt::format(", waiting: {}", user_waiting_queries)); auto running_query = user_process_list->second.queries.find(client_info.current_query_id); @@ -745,4 +755,69 @@ ProcessList::QueryAmount ProcessList::getQueryKindAmount(const IAST::QueryKind & return found->second; } +void ProcessList::increaseWaitingQueryAmount(const QueryStatusPtr & status) +{ + UInt64 limit = max_waiting_queries_amount.load(); + UInt64 value = waiting_queries_amount.load(); + while (true) + { + if (value >= limit) + throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous waiting queries. Maximum: {}, waiting: {}", + limit, value); + if (waiting_queries_amount.compare_exchange_strong(value, value + 1)) + break; + } + + // WARNING: After this point we should not throw, otherwise corresponding `decreaseWaitingQueryAmount` will not be called. + + // Update query kind counters + if (status->query_kind == IAST::QueryKind::Insert) + waiting_insert_queries_amount.fetch_add(1); + if (status->query_kind == IAST::QueryKind::Select) + waiting_select_queries_amount.fetch_add(1); + + // Update per-user counter + status->getUserProcessList()->waiting_queries_amount.fetch_add(1); + + // We have to notify because some queries might be waiting on `have_space` + // and this query leaves its space by transitioning to waiting state + have_space.notify_all(); +} + +void ProcessList::decreaseWaitingQueryAmount(const QueryStatusPtr & status) +{ + if (status->getUserProcessList()->waiting_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong insert waiting query amount for user: decrease to negative"); + + if (status->query_kind == IAST::QueryKind::Insert && waiting_insert_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong insert waiting query amount: decrease to negative"); + + if (status->query_kind == IAST::QueryKind::Select && waiting_select_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong select waiting query amount: decrease to negative"); + + if (waiting_queries_amount.fetch_sub(1) == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong waiting query amount: decrease to negative"); +} + +void ProcessList::incrementWaiters() +{ + ContextPtr context = CurrentThread::getQueryContext(); + QueryStatusPtr status = context->getProcessListElement(); + + // Query became "waiting" with the first thread that waits + if (status->waiting_threads.fetch_add(1) == 0) + increaseWaitingQueryAmount(status); +} + +void ProcessList::decrementWaiters() +{ + ContextPtr context = CurrentThread::getQueryContext(); + QueryStatusPtr status = context->getProcessListElement(); + + // Query became "non-waiting" with the last thread that no longer waits + if (status->waiting_threads.fetch_sub(1) == 1) + decreaseWaitingQueryAmount(status); +} + } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 1c253f562e8..75350627698 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -42,10 +42,6 @@ class ThreadStatus; class ProcessListEntry; -/** List of currently executing queries. - * Also implements limit on their number. - */ - /** Information of process list element. * To output in SHOW PROCESSLIST query. Does not contain any complex objects, that do something on copy or destructor. */ @@ -114,8 +110,13 @@ protected: /// Including EndOfStream or Exception. std::atomic is_all_data_sent { false }; + /// Number of threads for the query that are waiting for load jobs + std::atomic waiting_threads{0}; + + /// For initialization of ProcessListForUser during process insertion. void setUserProcessList(ProcessListForUser * user_process_list_); /// Be careful using it. For example, queries field of ProcessListForUser could be modified concurrently. + ProcessListForUser * getUserProcessList() { return user_process_list; } const ProcessListForUser * getUserProcessList() const { return user_process_list; } /// Sets an entry in the ProcessList associated with this QueryStatus. @@ -283,6 +284,9 @@ struct ProcessListForUser /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; + /// Number of queries waiting on load jobs + std::atomic waiting_queries_amount{0}; + ProcessListForUserInfo getInfo(bool get_profile_events = false) const; /// Clears MemoryTracker for the user. @@ -341,6 +345,9 @@ protected: }; +/** List of currently executing queries. + * Also implements limit on their number. + */ class ProcessList : public ProcessListBase { public: @@ -399,10 +406,21 @@ protected: /// amount of queries by query kind. QueryKindAmounts query_kind_amounts; + /// limit for waiting queries. 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. + std::atomic max_waiting_queries_amount{0}; + + /// amounts of waiting queries + std::atomic waiting_queries_amount{0}; + std::atomic waiting_insert_queries_amount{0}; + std::atomic waiting_select_queries_amount{0}; + void increaseQueryKindAmount(const IAST::QueryKind & query_kind); void decreaseQueryKindAmount(const IAST::QueryKind & query_kind); QueryAmount getQueryKindAmount(const IAST::QueryKind & query_kind) const; + void increaseWaitingQueryAmount(const QueryStatusPtr & status); + void decreaseWaitingQueryAmount(const QueryStatusPtr & status); + public: using EntryPtr = std::shared_ptr; @@ -458,6 +476,21 @@ public: return max_select_queries_amount; } + void setMaxWaitingQueriesAmount(UInt64 max_waiting_queries_amount_) + { + max_waiting_queries_amount.store(max_waiting_queries_amount_); + // NOTE: We cannot cancel waiting queries when limit is lowered. They have to wait anyways, but new queries will be canceled instead of waiting. + } + + size_t getMaxWaitingQueriesAmount() const + { + return max_waiting_queries_amount.load(); + } + + // Handlers for AsyncLoader waiters + void incrementWaiters(); + void decrementWaiters(); + /// Try call cancel() for input and output streams of query with specified id and user CancellationCode sendCancelToQuery(const String & current_query_id, const String & current_user, bool kill = false); CancellationCode sendCancelToQuery(QueryStatusPtr elem, bool kill = false); diff --git a/src/Storages/System/StorageSystemServerSettings.cpp b/src/Storages/System/StorageSystemServerSettings.cpp index f390985546b..bf14f757a19 100644 --- a/src/Storages/System/StorageSystemServerSettings.cpp +++ b/src/Storages/System/StorageSystemServerSettings.cpp @@ -70,6 +70,7 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context {"max_concurrent_queries", {std::to_string(context->getProcessList().getMaxSize()), ChangeableWithoutRestart::Yes}}, {"max_concurrent_insert_queries", {std::to_string(context->getProcessList().getMaxInsertQueriesAmount()), ChangeableWithoutRestart::Yes}}, {"max_concurrent_select_queries", {std::to_string(context->getProcessList().getMaxSelectQueriesAmount()), ChangeableWithoutRestart::Yes}}, + {"max_waiting_queries", {std::to_string(context->getProcessList().getMaxWaitingQueriesAmount()), ChangeableWithoutRestart::Yes}}, {"background_buffer_flush_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundBufferFlushSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}}, {"background_schedule_pool_size", {std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundSchedulePoolSize)), ChangeableWithoutRestart::IncreaseOnly}}, From 38cbc2c6c40541cc916bc591cd68b7eef70b1162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 7 Mar 2024 18:57:16 +0100 Subject: [PATCH 061/332] Restore digits --- base/base/itoa.cpp | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 9fefc9f0f07..ef844ff68a8 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -34,42 +34,37 @@ template -int digits10(T x) +inline int digits10(T x) { - if (x < T(10ULL)) + if (x < 10ULL) return 1; - if (x < T(100ULL)) + if (x < 100ULL) return 2; - if constexpr (sizeof(T) == 1) + if (x < 1000ULL) return 3; - else + + if (x < 1000000000000ULL) { - if (x < T(1000ULL)) - return 3; - - if (x < T(1000000000000ULL)) + if (x < 100000000ULL) { - if (x < T(100000000ULL)) + if (x < 1000000ULL) { - if (x < T(1000000ULL)) - { - if (x < T(10000ULL)) - return 4; - else - return 5 + (x >= T(100000ULL)); - } - - return 7 + (x >= T(10000000ULL)); + if (x < 10000ULL) + return 4; + else + return 5 + (x >= 100000ULL); } - if (x < T(10000000000ULL)) - return 9 + (x >= T(1000000000ULL)); - - return 11 + (x >= T(100000000000ULL)); + return 7 + (x >= 10000000ULL); } - return 12 + digits10(x / T(1000000000000ULL)); + if (x < 10000000000ULL) + return 9 + (x >= 1000000000ULL); + + return 11 + (x >= 100000000000ULL); } + + return 12 + digits10(x / 1000000000000ULL); } From 444595ac576438c9d0a259debf776187ddd3fcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 7 Mar 2024 20:00:07 +0100 Subject: [PATCH 062/332] More speed please Mr. compiler --- base/base/itoa.cpp | 165 +++++++++++++++++++++++++++++++-------------- 1 file changed, 114 insertions(+), 51 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index ef844ff68a8..08912edf3ea 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -72,7 +72,7 @@ namespace { template -static constexpr T pow10(size_t x) +ALWAYS_INLINE inline constexpr T pow10(size_t x) { return x ? 10 * pow10(x - 1) : 1; } @@ -143,7 +143,7 @@ struct QuotientAndRemainder }; template -QuotientAndRemainder static inline split(UnsignedOfSize value) +QuotientAndRemainder inline split(UnsignedOfSize value) { constexpr DivisionBy10PowN division; @@ -154,7 +154,7 @@ QuotientAndRemainder static inline split(UnsignedOfSize value) } -static inline char * outDigit(char * p, uint8_t value) +ALWAYS_INLINE inline char * outDigit(char * p, uint8_t value) { *p = '0' + value; ++p; @@ -176,7 +176,7 @@ static const char digits[201] = "00010203040506070809" "80818283848586878889" "90919293949596979899"; -static inline char * outTwoDigits(char * p, uint8_t value) +ALWAYS_INLINE inline char * outTwoDigits(char * p, uint8_t value) { memcpy(p, &digits[value * 2], 2); p += 2; @@ -187,9 +187,9 @@ static inline char * outTwoDigits(char * p, uint8_t value) namespace convert { template -static char * head(char * p, UInt u); +char * head(char * p, UInt u); template -static char * tail(char * p, UInt u); +char * tail(char * p, UInt u); //===----------------------------------------------------------===// // head: find most significant digit, skip leading zeros @@ -198,7 +198,7 @@ static char * tail(char * p, UInt u); // "x" contains quotient and remainder after division by 10^N // quotient is less than 10^N template -static inline char * head(char * p, QuotientAndRemainder x) +ALWAYS_INLINE inline char * head(char * p, QuotientAndRemainder x) { p = head(p, UnsignedOfSize(x.quotient)); p = tail(p, x.remainder); @@ -207,14 +207,14 @@ static inline char * head(char * p, QuotientAndRemainder x) // "u" is less than 10^2*N template -static inline char * head(char * p, UInt u) +ALWAYS_INLINE inline char * head(char * p, UInt u) { return u < pow10>(N) ? head(p, UnsignedOfSize(u)) : head(p, split(u)); } // recursion base case, selected when "u" is one byte template <> -inline char * head, 1>(char * p, UnsignedOfSize<1> u) +ALWAYS_INLINE inline char * head, 1>(char * p, UnsignedOfSize<1> u) { return u < 10 ? outDigit(p, u) : outTwoDigits(p, u); } @@ -225,7 +225,7 @@ inline char * head, 1>(char * p, UnsignedOfSize<1> u) // recursive step, "u" is less than 10^2*N template -static inline char * tail(char * p, UInt u) +ALWAYS_INLINE inline char * tail(char * p, UInt u) { QuotientAndRemainder x = split(u); p = tail(p, UnsignedOfSize(x.quotient)); @@ -235,7 +235,7 @@ static inline char * tail(char * p, UInt u) // recursion base case, selected when "u" is one byte template <> -inline char * tail, 1>(char * p, UnsignedOfSize<1> u) +ALWAYS_INLINE inline char * tail, 1>(char * p, UnsignedOfSize<1> u) { return outTwoDigits(p, u); } @@ -244,9 +244,8 @@ inline char * tail, 1>(char * p, UnsignedOfSize<1> u) // large values are >= 10^2*N // where x contains quotient and remainder after division by 10^N //===----------------------------------------------------------===// - template -static inline char * large(char * p, QuotientAndRemainder x) +ALWAYS_INLINE inline char * large(char * p, QuotientAndRemainder x) { QuotientAndRemainder y = split(x.quotient); p = head(p, UnsignedOfSize(y.quotient)); @@ -259,9 +258,8 @@ static inline char * large(char * p, QuotientAndRemainder x) // handle values of "u" that might be >= 10^2*N // where N is the size of "u" in bytes //===----------------------------------------------------------===// - template -static inline char * uitoa(char * p, UInt u) +ALWAYS_INLINE inline char * uitoa(char * p, UInt u) { if (u < pow10>(N)) return head(p, UnsignedOfSize(u)); @@ -272,7 +270,7 @@ static inline char * uitoa(char * p, UInt u) // selected when "u" is one byte template <> -inline char * uitoa, 1>(char * p, UnsignedOfSize<1> u) +ALWAYS_INLINE inline char * uitoa, 1>(char * p, UnsignedOfSize<1> u) { if (u < 10) return outDigit(p, u); @@ -292,14 +290,14 @@ inline char * uitoa, 1>(char * p, UnsignedOfSize<1> u) // itoa: handle unsigned integral operands (selected by SFINAE) template && std::is_integral_v> * = nullptr> -static inline char * itoa(U u, char * p) +ALWAYS_INLINE inline char * itoa(U u, char * p) { return convert::uitoa(p, u); } // itoa: handle signed integral operands (selected by SFINAE) template && std::is_integral_v> * = nullptr> -static inline char * itoa(I i, char * p) +ALWAYS_INLINE inline char * itoa(I i, char * p) { // Need "mask" to be filled with a copy of the sign bit. // If "i" is a negative value, then the result of "operator >>" @@ -335,63 +333,128 @@ static inline char * itoa(I i, char * p) } -template -static NO_INLINE char * writeUIntText(T _x, char * p) +const uint64_t max_multiple_of_hundred_that_fits_in_64_bits = 1'00'00'00'00'00'00'00'00'00ull; +constexpr int max_multiple_of_hundred_blocks = 9; +static_assert(max_multiple_of_hundred_that_fits_in_64_bits % 100 == 0); + +ALWAYS_INLINE inline char * writeUIntText(UInt128 _x, char * p) { - static_assert(std::is_same_v || std::is_same_v); - using T_ = std::conditional_t< - std::is_same_v, - unsigned __int128, + /// If we the highest 8 byte item is empty, we can print only the lowest item as i64 + if (_x.items[UInt128::_impl::little(1)] == 0) + return convert::itoa(_x.items[UInt128::_impl::little(0)], p); + + /// Doing operations using __int128 is faster, as we already rely on this feature + using T = unsigned __int128; + T x = (T(_x.items[UInt128::_impl::little(1)]) << 64) + T(_x.items[UInt128::_impl::little(0)]); + + /// We are going to accumulate blocks of 2 digits to print until the number is small enough to be printed as u64 + /// To do this we could do: x / 100, x % 100 + /// But this is too many iterations with long integers, so instead we can divide by a much longer integer + /// max_multiple_of_hundred_that_fits_in_64_bits and then get the blocks out of this (as u64) + static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits; + static const T largest_uint64 = std::numeric_limits::max(); + uint8_t two_values[20] = {0}; // 39 Max characters / 2 + + int current_block = 0; + while (x > largest_uint64) + { + uint64_t remainder = uint64_t(x % large_divisor); + x /= large_divisor; + + int pos = current_block; + while (remainder) + { + two_values[pos] = uint8_t(remainder % 100); + pos++; + remainder /= 100; + } + current_block += max_multiple_of_hundred_blocks; + } + + char * highest_part_print = convert::itoa(uint64_t(x), p); + for (int i = 0; i < current_block; i++) + { + outTwoDigits(highest_part_print, two_values[current_block - 1 - i]); + highest_part_print += 2; + } + + return highest_part_print; +} + +ALWAYS_INLINE inline char * writeUIntText(UInt256 _x, char * p) +{ + /// If possible, treat it as a smaller integer as they are much faster to print + if (_x.items[UInt256::_impl::little(3)] == 0 && _x.items[UInt256::_impl::little(2)] == 0) + return writeUIntText(UInt128{_x.items[UInt256::_impl::little(0)], _x.items[UInt256::_impl::little(1)]}, p); + + /// If available (x86) we transform from our custom class to _BitInt(256) which has better support in the compiler + /// and produces better code + using T = #if defined(__x86_64__) # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wbit-int-extension" unsigned _BitInt(256) # pragma clang diagnostic pop #else - T + UInt256 #endif - >; + ; - T_ x; - T_ hundred(100ULL); - if constexpr (std::is_same_v) - { - x = (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); - } - else - { #if defined(__x86_64__) - x = (T_(_x.items[T::_impl::little(3)]) << 192) + (T_(_x.items[T::_impl::little(2)]) << 128) - + (T_(_x.items[T::_impl::little(1)]) << 64) + T_(_x.items[T::_impl::little(0)]); + T x = (T(_x.items[UInt256::_impl::little(3)]) << 192) + (T(_x.items[UInt256::_impl::little(2)]) << 128) + + (T(_x.items[UInt256::_impl::little(1)]) << 64) + T(_x.items[UInt256::_impl::little(0)]); #else - x = _x; + T x = _x; #endif + + /// Similar to writeUIntText(UInt128) only that in this case we will stop as soon as we reach the largest u128 + /// and switch to that function + uint8_t two_values[39] = {0}; // 78 Max characters / 2 + int current_pos = 0; + + static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits; + static const T largest_uint128 = T(std::numeric_limits::max()) << 64 | T(std::numeric_limits::max()); + + while (x > largest_uint128) + { + uint64_t remainder = uint64_t(x % large_divisor); + x /= large_divisor; + + int pos = current_pos; + while (remainder) + { + two_values[pos] = uint8_t(remainder % 100); + pos++; + remainder /= 100; + } + current_pos += max_multiple_of_hundred_blocks; } - int len = digits10(x); - auto * pp = p + len; - while (x >= hundred) +#if defined(__x86_64__) + UInt128 pending{uint64_t(x), uint64_t(x >> 64)}; +#else + UInt128 pending{x.items[UInt256::_impl::little(0)], x.items[UInt256::_impl::little(1)]}; +#endif + + char * highest_part_print = writeUIntText(pending, p); + for (int i = 0; i < current_pos; i++) { - const auto i = x % hundred; - x /= hundred; - pp -= 2; - outTwoDigits(pp, i); + outTwoDigits(highest_part_print, two_values[current_pos - 1 - i]); + highest_part_print += 2; } - if (x < 10) - *p = '0' + x; - else - outTwoDigits(p, x); - return p + len; + + return highest_part_print; } -static ALWAYS_INLINE inline char * writeLeadingMinus(char * pos) + +ALWAYS_INLINE inline char * writeLeadingMinus(char * pos) { *pos = '-'; return pos + 1; } template -static ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) +ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) { static_assert(std::is_same_v || std::is_same_v); From b0b38121a60682ee79d47914e18dd57d892f394e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 7 Mar 2024 20:44:03 +0000 Subject: [PATCH 063/332] Crash repro --- src/Storages/StorageMergeTree.cpp | 1 + ...eplicas_cte_explain_syntax_crash.reference | 15 +++++++ ...llel_replicas_cte_explain_syntax_crash.sql | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference create mode 100644 tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index e15b308f084..dbc901d4a5f 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -220,6 +220,7 @@ void StorageMergeTree::read( Block header; if (local_context->getSettingsRef().allow_experimental_analyzer) { + chassert(query_info.query_tree); QueryTreeNodePtr modified_query_tree = query_info.query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, local_context); modified_query_tree = buildQueryTreeForShard(query_info.planner_context, modified_query_tree); diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference new file mode 100644 index 00000000000..078890722fd --- /dev/null +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.reference @@ -0,0 +1,15 @@ +WITH + cte1 AS + ( + SELECT n + FROM numbers_1e6__fuzz_34 + ), + cte2 AS + ( + SELECT n + FROM numbers_1e6__fuzz_33 + PREWHERE n IN cte1 + ) +SELECT count() +FROM +cte2 diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql new file mode 100644 index 00000000000..a407fceb1c6 --- /dev/null +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -0,0 +1,44 @@ +DROP TABLE IF EXISTS numbers_1e6__fuzz_34; +DROP TABLE IF EXISTS numbers_1e6__fuzz_33; + +CREATE TABLE numbers_1e6__fuzz_34 +( + `n` LowCardinality(Nullable(UInt8)) +) +ENGINE = MergeTree +ORDER BY n +SETTINGS allow_nullable_key = 1 +AS SELECT * +FROM numbers(1000000) +SETTINGS allow_suspicious_low_cardinality_types = 1; + + +CREATE TABLE numbers_1e6__fuzz_33 +( + `n` LowCardinality(Nullable(UInt8)) +) +ENGINE = MergeTree +ORDER BY n +SETTINGS allow_nullable_key = 1 +AS SELECT * +FROM numbers(1000000) +SETTINGS allow_suspicious_low_cardinality_types = 1; + +SET allow_experimental_analyzer = 0; +SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +EXPLAIN SYNTAX +WITH + cte1 AS + ( + SELECT n + FROM numbers_1e6__fuzz_34 + ), + cte2 AS + ( + SELECT n + FROM numbers_1e6__fuzz_33 + PREWHERE n IN (cte1) + ) +SELECT count() +FROM cte2; +-- SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; From cde811804f68656577dd1497e511a4a9295d4544 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 8 Mar 2024 12:20:34 +0800 Subject: [PATCH 064/332] prevent memcpySmallAllowReadWriteOverflow15Impl optimized to memcpy --- src/Common/memcpySmall.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/memcpySmall.h b/src/Common/memcpySmall.h index f3d26c60380..f5e9f31fc46 100644 --- a/src/Common/memcpySmall.h +++ b/src/Common/memcpySmall.h @@ -49,6 +49,9 @@ namespace detail dst += 16; src += 16; n -= 16; + + /// Avoid clang loop-idion optimization, which transforms _mm_storeu_si128 to built-in memcpy + __asm__ __volatile__("" : : : "memory"); } } } From 89ae39e598a481dbb2c610ee7dca1fc7272517b7 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 8 Mar 2024 12:22:07 +0800 Subject: [PATCH 065/332] optimize column string replicate --- src/Columns/ColumnString.cpp | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index b9128372cea..0c52a7be086 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -460,6 +460,7 @@ void ColumnString::updatePermutationWithCollation(const Collator & collator, Per DefaultPartialSort()); } + ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const { size_t col_size = size(); @@ -471,32 +472,35 @@ ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const if (0 == col_size) return res; - Chars & res_chars = res->chars; Offsets & res_offsets = res->offsets; - res_chars.reserve_exact(chars.size() / col_size * replicate_offsets.back()); - res_offsets.reserve_exact(replicate_offsets.back()); - - Offset prev_replicate_offset = 0; - Offset prev_string_offset = 0; - Offset current_new_offset = 0; + res_offsets.resize_exact(replicate_offsets.back()); + Chars & res_chars = res->chars; + size_t res_chars_size = 0; for (size_t i = 0; i < col_size; ++i) { - size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset; - size_t string_size = offsets[i] - prev_string_offset; + size_t size_to_replicate = replicate_offsets[i] - replicate_offsets[i - 1]; + size_t string_size = offsets[i] - offsets[i - 1]; + res_chars_size += size_to_replicate * string_size; + } + res_chars.resize_exact(res_chars_size); + size_t curr_row = 0; + size_t curr_offset = 0; + for (size_t i = 0; i < col_size; ++i) + { + const size_t size_to_replicate = replicate_offsets[i] - replicate_offsets[i - 1]; + const size_t string_size = offsets[i] - offsets[i-1]; + const UInt8 * src = &chars[offsets[i - 1]]; for (size_t j = 0; j < size_to_replicate; ++j) { - current_new_offset += string_size; - res_offsets.push_back(current_new_offset); - - res_chars.resize(res_chars.size() + string_size); memcpySmallAllowReadWriteOverflow15( - &res_chars[res_chars.size() - string_size], &chars[prev_string_offset], string_size); - } + &res_chars[curr_offset], src, string_size); - prev_replicate_offset = replicate_offsets[i]; - prev_string_offset = offsets[i]; + curr_offset += string_size; + res_offsets[curr_row] = curr_offset; + ++curr_row; + } } return res; From 391af00b64e62dc7d9e1c5726c1aa202f19dffd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=89=AC?= <654010905@qq.com> Date: Fri, 8 Mar 2024 13:47:35 +0800 Subject: [PATCH 066/332] Update memcpySmall.h --- src/Common/memcpySmall.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/memcpySmall.h b/src/Common/memcpySmall.h index f5e9f31fc46..90648254d76 100644 --- a/src/Common/memcpySmall.h +++ b/src/Common/memcpySmall.h @@ -50,7 +50,7 @@ namespace detail src += 16; n -= 16; - /// Avoid clang loop-idion optimization, which transforms _mm_storeu_si128 to built-in memcpy + /// Avoid clang loop-idiom optimization, which transforms _mm_storeu_si128 to built-in memcpy __asm__ __volatile__("" : : : "memory"); } } From d52027c5a09f0f9619bc2f5df639f1a042b2c084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 12:00:34 +0100 Subject: [PATCH 067/332] Style and comments --- base/base/itoa.cpp | 47 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 08912edf3ea..4475ae416b9 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -165,16 +165,16 @@ ALWAYS_INLINE inline char * outDigit(char * p, uint8_t value) // into ascii characters as described by Andrei Alexandrescu in // https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/ -static const char digits[201] = "00010203040506070809" - "10111213141516171819" - "20212223242526272829" - "30313233343536373839" - "40414243444546474849" - "50515253545556575859" - "60616263646566676869" - "70717273747576777879" - "80818283848586878889" - "90919293949596979899"; +const char digits[201] = "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899"; ALWAYS_INLINE inline char * outTwoDigits(char * p, uint8_t value) { @@ -334,23 +334,24 @@ ALWAYS_INLINE inline char * itoa(I i, char * p) const uint64_t max_multiple_of_hundred_that_fits_in_64_bits = 1'00'00'00'00'00'00'00'00'00ull; -constexpr int max_multiple_of_hundred_blocks = 9; +const int max_multiple_of_hundred_blocks = 9; static_assert(max_multiple_of_hundred_that_fits_in_64_bits % 100 == 0); ALWAYS_INLINE inline char * writeUIntText(UInt128 _x, char * p) { - /// If we the highest 8 byte item is empty, we can print only the lowest item as i64 + /// If we the highest 64bit item is empty, we can print just the lowest item as u64 if (_x.items[UInt128::_impl::little(1)] == 0) return convert::itoa(_x.items[UInt128::_impl::little(0)], p); - /// Doing operations using __int128 is faster, as we already rely on this feature + /// Doing operations using __int128 is faster and we already rely on this feature using T = unsigned __int128; T x = (T(_x.items[UInt128::_impl::little(1)]) << 64) + T(_x.items[UInt128::_impl::little(0)]); /// We are going to accumulate blocks of 2 digits to print until the number is small enough to be printed as u64 /// To do this we could do: x / 100, x % 100 - /// But this is too many iterations with long integers, so instead we can divide by a much longer integer - /// max_multiple_of_hundred_that_fits_in_64_bits and then get the blocks out of this (as u64) + /// But these would mean doing many iterations with long integers, so instead we divide by a much longer integer + /// multiple of 100 (100^9) and then get the blocks out of it (as u64) + /// Once we reach u64::max we can stop and use the fast method to print that in the front static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits; static const T largest_uint64 = std::numeric_limits::max(); uint8_t two_values[20] = {0}; // 39 Max characters / 2 @@ -358,15 +359,15 @@ ALWAYS_INLINE inline char * writeUIntText(UInt128 _x, char * p) int current_block = 0; while (x > largest_uint64) { - uint64_t remainder = uint64_t(x % large_divisor); + uint64_t u64_remainder = uint64_t(x % large_divisor); x /= large_divisor; int pos = current_block; - while (remainder) + while (u64_remainder) { - two_values[pos] = uint8_t(remainder % 100); + two_values[pos] = uint8_t(u64_remainder % 100); pos++; - remainder /= 100; + u64_remainder /= 100; } current_block += max_multiple_of_hundred_blocks; } @@ -417,15 +418,15 @@ ALWAYS_INLINE inline char * writeUIntText(UInt256 _x, char * p) while (x > largest_uint128) { - uint64_t remainder = uint64_t(x % large_divisor); + uint64_t u64_remainder = uint64_t(x % large_divisor); x /= large_divisor; int pos = current_pos; - while (remainder) + while (u64_remainder) { - two_values[pos] = uint8_t(remainder % 100); + two_values[pos] = uint8_t(u64_remainder % 100); pos++; - remainder /= 100; + u64_remainder /= 100; } current_pos += max_multiple_of_hundred_blocks; } From 2f3c103367de4fab57602e3fc1821608df718c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 12:02:53 +0100 Subject: [PATCH 068/332] OSX quirks --- base/base/itoa.cpp | 4 ++++ base/base/itoa.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 4475ae416b9..d877f15b563 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -542,6 +542,10 @@ char * itoa(Int256 i, char * p) #define INSTANTIATION(T) template char * itoa(T i, char * p); FOR_MISSING_INTEGER_TYPES(INSTANTIATION) +#if defined(OS_DARWIN) +INSTANTIATION(size_t) +#endif + #undef FOR_MISSING_INTEGER_TYPES #undef INSTANTIATION diff --git a/base/base/itoa.h b/base/base/itoa.h index 71603cdeb88..98a570b12fa 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -25,6 +25,10 @@ template <> char * itoa(Int256 i, char * p); extern template char * itoa(T i, char * p); FOR_MISSING_INTEGER_TYPES(INSTANTIATION) +#if defined(OS_DARWIN) +INSTANTIATION(size_t) +#endif + #undef FOR_MISSING_INTEGER_TYPES #undef INSTANTIATION From e7cc49212a15ca8bcf87950225e066d04c8823e4 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:28:14 +0100 Subject: [PATCH 069/332] fix tidy build --- src/Functions/array/arrayDistance.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 03f0bc7b286..0045075ddef 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -523,18 +523,18 @@ private: const auto & offsets_y = array_y.getOffsets(); ColumnArray::Offset prev_offset = 0; - for (size_t row = 0; row < offsets_y.size(); ++row) + for (auto offset_y : offsets_y) { - if (offsets_x[0] != offsets_y[row] - prev_offset) [[unlikely]] + if (offsets_x[0] != offset_y - prev_offset) [[unlikely]] { throw Exception( ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arguments of function {} have different array sizes: {} and {}", getName(), offsets_x[0], - offsets_y[row] - prev_offset); + offset_y - prev_offset); } - prev_offset = offsets_y[row]; + prev_offset = offset_y; } const typename Kernel::ConstParams kernel_params = initConstParams(arguments); From 07ce390609238318a7ab115b3d7020f03150ce76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 13:59:24 +0100 Subject: [PATCH 070/332] No public templates, just happiness --- base/base/itoa.cpp | 44 +++++++++++++++++++++----------------------- base/base/itoa.h | 35 ++++++++++++++--------------------- 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index d877f15b563..3c4f0bb048d 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -32,9 +32,11 @@ #include #include +namespace +{ template -inline int digits10(T x) +int digits10T(T x) { if (x < 10ULL) return 1; @@ -64,13 +66,9 @@ inline int digits10(T x) return 11 + (x >= 100000000000ULL); } - return 12 + digits10(x / 1000000000000ULL); + return 12 + digits10T(x / 1000000000000ULL); } - -namespace -{ - template ALWAYS_INLINE inline constexpr T pow10(size_t x) { @@ -487,67 +485,62 @@ ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) } } -template -char * itoa(T i, char * p) -{ - return convert::itoa(i, p); -} -template <> char * itoa(UInt8 i, char * p) { return convert::itoa(uint8_t(i), p); } -template <> char * itoa(Int8 i, char * p) { return convert::itoa(int8_t(i), p); } -template <> char * itoa(UInt128 i, char * p) { return writeUIntText(i, p); } -template <> char * itoa(Int128 i, char * p) { return writeSIntText(i, p); } -template <> char * itoa(UInt256 i, char * p) { return writeUIntText(i, p); } -template <> char * itoa(Int256 i, char * p) { return writeSIntText(i, p); } +#define DEFAULT_ITOA(T) \ + char * itoa(T i, char * p) \ + { \ + return convert::itoa(i, p); \ + } + #define FOR_MISSING_INTEGER_TYPES(M) \ - M(int8_t) \ M(uint8_t) \ M(UInt16) \ M(UInt32) \ M(UInt64) \ + M(int8_t) \ M(Int16) \ M(Int32) \ M(Int64) -#define INSTANTIATION(T) template char * itoa(T i, char * p); -FOR_MISSING_INTEGER_TYPES(INSTANTIATION) +FOR_MISSING_INTEGER_TYPES(DEFAULT_ITOA) #if defined(OS_DARWIN) -INSTANTIATION(size_t) +DEFAULT_ITOA(unsigned long) +DEFAULT_ITOA(long) #endif #undef FOR_MISSING_INTEGER_TYPES -#undef INSTANTIATION +#undef DEFAULT_ITOA #define DIGITS_INTEGER_TYPES(M) \ @@ -559,7 +552,12 @@ INSTANTIATION(size_t) M(UInt128) \ M(UInt256) -#define INSTANTIATION(T) template int digits10(T x); +#define INSTANTIATION(T) \ + int digits10(T x) \ + { \ + return digits10T(x); \ + } + DIGITS_INTEGER_TYPES(INSTANTIATION) #undef DIGITS_INTEGER_TYPES diff --git a/base/base/itoa.h b/base/base/itoa.h index 98a570b12fa..9a89fa739dd 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -2,39 +2,33 @@ #include -template char * itoa(T i, char * p); - -template <> char * itoa(UInt8 i, char * p); -template <> char * itoa(Int8 i, char * p); -template <> char * itoa(UInt128 i, char * p); -template <> char * itoa(Int128 i, char * p); -template <> char * itoa(UInt256 i, char * p); -template <> char * itoa(Int256 i, char * p); - -#define FOR_MISSING_INTEGER_TYPES(M) \ - M(int8_t) \ +#define FOR_INTEGER_TYPES(M) \ M(uint8_t) \ + M(UInt8) \ M(UInt16) \ M(UInt32) \ M(UInt64) \ + M(UInt128) \ + M(UInt256) \ + M(int8_t) \ + M(Int8) \ M(Int16) \ M(Int32) \ - M(Int64) + M(Int64) \ + M(Int128) \ + M(Int256) -#define INSTANTIATION(T) \ - extern template char * itoa(T i, char * p); -FOR_MISSING_INTEGER_TYPES(INSTANTIATION) +#define INSTANTIATION(T) char * itoa(T i, char * p); +FOR_INTEGER_TYPES(INSTANTIATION) #if defined(OS_DARWIN) -INSTANTIATION(size_t) +INSTANTIATION(unsigned long) +INSTANTIATION(long) #endif #undef FOR_MISSING_INTEGER_TYPES #undef INSTANTIATION - -template int digits10(T x); - #define DIGITS_INTEGER_TYPES(M) \ M(uint8_t) \ M(UInt8) \ @@ -43,8 +37,7 @@ template int digits10(T x); M(UInt64) \ M(UInt128) \ M(UInt256) -#define INSTANTIATION(T) \ - extern template int digits10(T x); +#define INSTANTIATION(T) int digits10(T x); DIGITS_INTEGER_TYPES(INSTANTIATION) #undef DIGITS_INTEGER_TYPES #undef INSTANTIATION From 4b964979c2c831a2ddeee5a7b0c10066f520ab11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 14:22:33 +0100 Subject: [PATCH 071/332] Digits doesn't belong with itoa anymore --- base/base/itoa.cpp | 55 ----------------------------------- base/base/itoa.h | 13 --------- src/Functions/countDigits.cpp | 34 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 68 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 3c4f0bb048d..608258c6b56 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -35,40 +35,6 @@ namespace { -template -int digits10T(T x) -{ - if (x < 10ULL) - return 1; - if (x < 100ULL) - return 2; - if (x < 1000ULL) - return 3; - - if (x < 1000000000000ULL) - { - if (x < 100000000ULL) - { - if (x < 1000000ULL) - { - if (x < 10000ULL) - return 4; - else - return 5 + (x >= 100000ULL); - } - - return 7 + (x >= 10000000ULL); - } - - if (x < 10000000000ULL) - return 9 + (x >= 1000000000ULL); - - return 11 + (x >= 100000000000ULL); - } - - return 12 + digits10T(x / 1000000000000ULL); -} - template ALWAYS_INLINE inline constexpr T pow10(size_t x) { @@ -541,24 +507,3 @@ DEFAULT_ITOA(long) #undef FOR_MISSING_INTEGER_TYPES #undef DEFAULT_ITOA - - -#define DIGITS_INTEGER_TYPES(M) \ - M(uint8_t) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(UInt128) \ - M(UInt256) - -#define INSTANTIATION(T) \ - int digits10(T x) \ - { \ - return digits10T(x); \ - } - -DIGITS_INTEGER_TYPES(INSTANTIATION) - -#undef DIGITS_INTEGER_TYPES -#undef INSTANTIATION diff --git a/base/base/itoa.h b/base/base/itoa.h index 9a89fa739dd..e69ce0ef17d 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -28,16 +28,3 @@ INSTANTIATION(long) #undef FOR_MISSING_INTEGER_TYPES #undef INSTANTIATION - -#define DIGITS_INTEGER_TYPES(M) \ - M(uint8_t) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(UInt128) \ - M(UInt256) -#define INSTANTIATION(T) int digits10(T x); -DIGITS_INTEGER_TYPES(INSTANTIATION) -#undef DIGITS_INTEGER_TYPES -#undef INSTANTIATION diff --git a/src/Functions/countDigits.cpp b/src/Functions/countDigits.cpp index 2ca8d944b0a..f2712b5b301 100644 --- a/src/Functions/countDigits.cpp +++ b/src/Functions/countDigits.cpp @@ -20,6 +20,40 @@ namespace ErrorCodes namespace { +template +int digits10(T x) +{ + if (x < 10ULL) + return 1; + if (x < 100ULL) + return 2; + if (x < 1000ULL) + return 3; + + if (x < 1000000000000ULL) + { + if (x < 100000000ULL) + { + if (x < 1000000ULL) + { + if (x < 10000ULL) + return 4; + else + return 5 + (x >= 100000ULL); + } + + return 7 + (x >= 10000000ULL); + } + + if (x < 10000000000ULL) + return 9 + (x >= 1000000000ULL); + + return 11 + (x >= 100000000000ULL); + } + + return 12 + digits10(x / 1000000000000ULL); +} + /// Returns number of decimal digits you need to represent the value. /// For Decimal values takes in account their scales: calculates result over underlying int type which is (value * scale). /// countDigits(42) = 2, countDigits(42.000) = 5, countDigits(0.04200) = 4. From 4f27dd8f9c887cec6273dc7960b165ee2537ad26 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 8 Mar 2024 14:27:50 +0100 Subject: [PATCH 072/332] upd test --- .../0_stateless/02864_restore_table_with_broken_part.reference | 2 +- .../queries/0_stateless/02864_restore_table_with_broken_part.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02864_restore_table_with_broken_part.reference b/tests/queries/0_stateless/02864_restore_table_with_broken_part.reference index 9a8dcda81df..9247a7d6ab6 100644 --- a/tests/queries/0_stateless/02864_restore_table_with_broken_part.reference +++ b/tests/queries/0_stateless/02864_restore_table_with_broken_part.reference @@ -1,4 +1,4 @@ -data.bin doesn't exist: while restoring part all_2_2_0 +OK RESTORED 1 3 diff --git a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh index cf99c7e9284..d3252b29eb7 100755 --- a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh +++ b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh @@ -26,7 +26,7 @@ ln -s "$SRC_BACKUP_DIR/$SRC_BACKUP_FILENAME" "$BACKUPS_DIR/$BACKUP_FILENAME" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS tbl" # First try to restore with the setting `restore_broken_parts_as_detached` set to false. -$CLICKHOUSE_CLIENT --query "RESTORE TABLE default.tbl AS tbl FROM $BACKUP_NAME" 2>&1 | grep -o -m 1 "data.bin doesn't exist: while restoring part all_2_2_0" +$CLICKHOUSE_CLIENT --query "RESTORE TABLE default.tbl AS tbl FROM $BACKUP_NAME" 2>&1 | tr -d \\n | grep "data.bin doesn't exist" | grep "while restoring part all_2_2_0" > /dev/null && echo "OK" || echo "FAILED" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS tbl" From 45a1f0f8ed33fb09fee92137a391dfd25ba63f05 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 8 Mar 2024 14:02:34 +0000 Subject: [PATCH 073/332] More consistency and extended tests --- src/Functions/array/arrayDistance.cpp | 8 ++-- src/Functions/array/arrayDotProduct.cpp | 43 ++++++++----------- .../0_stateless/02708_dotProduct.reference | 12 ++++++ .../queries/0_stateless/02708_dotProduct.sql | 16 +++++-- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 0045075ddef..8b591e37ff6 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -471,10 +471,9 @@ private: const typename Kernel::ConstParams kernel_params = initConstParams(arguments); - auto result = ColumnVector::create(input_rows_count); - auto & result_data = result->getData(); + auto col_res = ColumnVector::create(input_rows_count); + auto & result_data = col_res->getData(); - /// Do the actual computation ColumnArray::Offset prev = 0; size_t row = 0; @@ -503,7 +502,7 @@ private: result_data[row] = Kernel::finalize(state, kernel_params); row++; } - return result; + return col_res; } /// Special case when the 1st parameter is Const @@ -542,7 +541,6 @@ private: auto result = ColumnVector::create(input_rows_count); auto & result_data = result->getData(); - /// Do the actual computation size_t prev = 0; size_t row = 0; diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 8b7c85e05dd..97dc9653bab 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -140,6 +140,7 @@ public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } size_t getNumberOfArguments() const override { return 2; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -174,13 +175,13 @@ public: ACTION(Float32) \ ACTION(Float64) - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { switch (result_type->getTypeId()) { #define ON_TYPE(type) \ case TypeIndex::type: \ - return executeWithResultType(arguments); \ + return executeWithResultType(arguments, input_rows_count); \ break; SUPPORTED_TYPES(ON_TYPE) @@ -193,7 +194,7 @@ public: private: template - ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments) const + ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); @@ -201,7 +202,7 @@ private: { #define ON_TYPE(type) \ case TypeIndex::type: \ - return executeWithResultTypeAndLeftType(arguments); \ + return executeWithResultTypeAndLeftType(arguments, input_rows_count); \ break; SUPPORTED_TYPES(ON_TYPE) @@ -218,7 +219,7 @@ private: } template - ColumnPtr executeWithResultTypeAndLeftType(const ColumnsWithTypeAndName & arguments) const + ColumnPtr executeWithResultTypeAndLeftType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); @@ -226,7 +227,7 @@ private: { #define ON_TYPE(type) \ case TypeIndex::type: \ - return executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column); \ + return executeWithResultTypeAndLeftTypeAndRightType(arguments[0].column, arguments[1].column, input_rows_count); \ break; SUPPORTED_TYPES(ON_TYPE) @@ -243,15 +244,15 @@ private: } template - ColumnPtr executeWithResultTypeAndLeftTypeAndRightType(ColumnPtr col_x, ColumnPtr col_y) const + ColumnPtr executeWithResultTypeAndLeftTypeAndRightType(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count) const { if (typeid_cast(col_x.get())) { - return executeWithLeftArgConst(col_x, col_y); + return executeWithLeftArgConst(col_x, col_y, input_rows_count); } else if (typeid_cast(col_y.get())) { - return executeWithLeftArgConst(col_y, col_x); + return executeWithLeftArgConst(col_y, col_x, input_rows_count); } col_x = col_x->convertToFullColumnIfConst(); @@ -268,16 +269,13 @@ private: if (!array_x.hasEqualOffsets(array_y)) throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Array arguments for function {} must have equal sizes", getName()); - auto col_res = ColumnVector::create(); - auto & result = col_res->getData(); - - size_t size = offsets_x.size(); - result.resize(size); + auto col_res = ColumnVector::create(input_rows_count); + auto & result_data = col_res->getData(); ColumnArray::Offset current_offset = 0; - for (size_t row = 0; row < size; ++row) + for (size_t row = 0; row < input_rows_count; ++row) { - size_t array_size = offsets_x[row] - current_offset; + const size_t array_size = offsets_x[row] - current_offset; size_t i = 0; @@ -298,7 +296,7 @@ private: for (; i < array_size; ++i) Kernel::template accumulate(state, static_cast(data_x[current_offset + i]), static_cast(data_y[current_offset + i])); - result[row] = Kernel::template finalize(state); + result_data[row] = Kernel::template finalize(state); current_offset = offsets_x[row]; } @@ -307,7 +305,7 @@ private: } template - ColumnPtr executeWithLeftArgConst(ColumnPtr col_x, ColumnPtr col_y) const + ColumnPtr executeWithLeftArgConst(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count) const { col_x = assert_cast(col_x.get())->getDataColumnPtr(); col_y = col_y->convertToFullColumnIfConst(); @@ -336,16 +334,13 @@ private: prev_offset = offset_y; } - auto col_res = ColumnVector::create(); + auto col_res = ColumnVector::create(input_rows_count); auto & result = col_res->getData(); - size_t size = offsets_y.size(); - result.resize(size); - ColumnArray::Offset current_offset = 0; - for (size_t row = 0; row < size; ++row) + for (size_t row = 0; row < input_rows_count; ++row) { - size_t array_size = offsets_x[0]; + const size_t array_size = offsets_x[0]; typename Kernel::template State state; size_t i = 0; diff --git a/tests/queries/0_stateless/02708_dotProduct.reference b/tests/queries/0_stateless/02708_dotProduct.reference index 593071a3521..93a67e4c0be 100644 --- a/tests/queries/0_stateless/02708_dotProduct.reference +++ b/tests/queries/0_stateless/02708_dotProduct.reference @@ -11,6 +11,8 @@ [-1,-2,-3] [4,5,6] -32 Int64 [1,2,3] [4,5,6] 32 Float32 [1,2,3] [4,5,6] 32 Float64 +[] [] 0 Float32 +[] [] 0 UInt16 -- Tuple (1,2,3) (4,5,6) 32 UInt64 (1,2,3) (4,5,6) 32 UInt64 @@ -24,6 +26,8 @@ (1,2,3) (4,5,6) 32 Float64 -- Non-const argument [1,2,3] [4,5,6] 32 UInt16 +[] [] 0 Float32 +[] [] 0 UInt16 -- Array with mixed element arguments types (result type is the supertype) [1,2,3] [4,5,6] 32 Float32 -- Tuple with mixed element arguments types @@ -33,8 +37,16 @@ 32 32 -- Tests that trigger special paths + -- non-const / non-const 0 61 1 186 +0 61 +1 186 +0 61 +1 186 + -- const / non-const +0 62 +1 187 0 62 1 187 0 62 diff --git a/tests/queries/0_stateless/02708_dotProduct.sql b/tests/queries/0_stateless/02708_dotProduct.sql index ac94ecc28d3..05c66777dff 100644 --- a/tests/queries/0_stateless/02708_dotProduct.sql +++ b/tests/queries/0_stateless/02708_dotProduct.sql @@ -19,6 +19,9 @@ SELECT [-1, -2, -3]::Array(Int32) AS x, [4, 5, 6]::Array(Int32) AS y, dotProduct SELECT [-1, -2, -3]::Array(Int64) AS x, [4, 5, 6]::Array(Int64) AS y, dotProduct(x, y) AS res, toTypeName(res); SELECT [1, 2, 3]::Array(Float32) AS x, [4, 5, 6]::Array(Float32) AS y, dotProduct(x, y) AS res, toTypeName(res); SELECT [1, 2, 3]::Array(Float64) AS x, [4, 5, 6]::Array(Float64) AS y, dotProduct(x, y) AS res, toTypeName(res); +-- empty arrays +SELECT []::Array(Float32) AS x, []::Array(Float32) AS y, dotProduct(x, y) AS res, toTypeName(res); +SELECT []::Array(UInt8) AS x, []::Array(UInt8) AS y, dotProduct(x, y) AS res, toTypeName(res); SELECT ' -- Tuple'; SELECT (1::UInt8, 2::UInt8, 3::UInt8) AS x, (4::UInt8, 5::UInt8, 6::UInt8) AS y, dotProduct(x, y) AS res, toTypeName(res); @@ -34,6 +37,8 @@ SELECT (1::Float64, 2::Float64, 3::Float64) AS x, (4::Float64, 5::Float64, 6::Fl SELECT '-- Non-const argument'; SELECT materialize([1::UInt8, 2::UInt8, 3::UInt8]) AS x, [4::UInt8, 5::UInt8, 6::UInt8] AS y, dotProduct(x, y) AS res, toTypeName(res); +SELECT materialize([]::Array(Float32)) AS x, []::Array(Float32) AS y, dotProduct(x, y) AS res, toTypeName(res); +SELECT materialize([]::Array(UInt8)) AS x, []::Array(UInt8) AS y, dotProduct(x, y) AS res, toTypeName(res); SELECT ' -- Array with mixed element arguments types (result type is the supertype)'; SELECT [1::UInt16, 2::UInt8, 3::Float32] AS x, [4::Int16, 5::Float32, 6::UInt8] AS y, dotProduct(x, y) AS res, toTypeName(res); @@ -50,7 +55,12 @@ SELECT '-- Tests that trigger special paths'; DROP TABLE IF EXISTS tab; CREATE TABLE tab(id UInt64, vec Array(Float32)) ENGINE = MergeTree ORDER BY id; INSERT INTO tab VALUES (0, [0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0, 3.0, 0.0, 1.0, 2.0]) (1, [5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]); -SELECT id, arrayDotProduct(vec, vec) FROM tab ORDER BY id; -- non-const / non-const -SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float32), vec) FROM tab ORDER BY id; -- const / non-const -SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float64), vec) FROM tab ORDER BY id; -- const / non-const +SELECT ' -- non-const / non-const'; +SELECT id, arrayDotProduct(vec, vec) FROM tab ORDER BY id; +SELECT id, arrayDotProduct(vec::Array(Float64), vec::Array(Float64)) FROM tab ORDER BY id; +SELECT id, arrayDotProduct(vec::Array(UInt32), vec::Array(UInt32)) FROM tab ORDER BY id; +SELECT ' -- const / non-const'; +SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float32), vec) FROM tab ORDER BY id; +SELECT id, arrayDotProduct([5.0, 2.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0, 3.0, 5.0, 1.0, 2.0]::Array(Float64), vec) FROM tab ORDER BY id; +SELECT id, arrayDotProduct([5, 2, 2, 3, 5, 1, 2, 3, 5, 1, 2, 3, 5, 1, 2, 3, 5, 1, 2]::Array(UInt32), vec) FROM tab ORDER BY id; DROP TABLE tab; From 45efa69189784ce65bffd0d84462dcb30c1e6bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 15:17:59 +0100 Subject: [PATCH 074/332] Add perf tests --- tests/performance/bigint_formatting.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/performance/bigint_formatting.xml diff --git a/tests/performance/bigint_formatting.xml b/tests/performance/bigint_formatting.xml new file mode 100644 index 00000000000..c3454f91d1d --- /dev/null +++ b/tests/performance/bigint_formatting.xml @@ -0,0 +1,13 @@ + + + CREATE TABLE bigint ( u128 UInt128, i128 Int128, u256 UInt256, i256 Int256) ENGINE = Memory + AS + SELECT * FROM generateRandom('u128 UInt128, i128 Int128, u256 UInt256, i256 Int256', 42) LIMIT 50000; + + DROP TABLE IF EXISTS bigint + + SELECT * FROM bigint WHERE NOT ignore(toString(u128)) SETTINGS max_threads = 1 + SELECT * FROM bigint WHERE NOT ignore(toString(i128)) SETTINGS max_threads = 1 + SELECT * FROM bigint WHERE NOT ignore(toString(u256)) SETTINGS max_threads = 1 + SELECT * FROM bigint WHERE NOT ignore(toString(i256)) SETTINGS max_threads = 1 + From e2317477f7b95d07407db8def968d286aa9e270d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 8 Mar 2024 17:12:31 +0100 Subject: [PATCH 075/332] fix removing is_active node after re-creation --- src/Databases/DatabaseReplicatedWorker.cpp | 2 ++ tests/integration/test_replicated_database/test.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 2056b403ff6..0a6e8f9345e 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -75,6 +75,8 @@ void DatabaseReplicatedDDLWorker::initializeReplication() String active_path = fs::path(database->replica_path) / "active"; String active_id = toString(ServerUUID::get()); zookeeper->deleteEphemeralNodeIfContentMatches(active_path, active_id); + if (active_node_holder) + active_node_holder->setAlreadyRemoved(); zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); active_node_holder.reset(); active_node_holder_zookeeper = zookeeper; diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index b47f86a843d..4f449f9a296 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1141,6 +1141,8 @@ def test_sync_replica(started_cluster): dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") + assert "2\n" == main_node.query("SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'") + assert dummy_node.query( "SELECT count() FROM system.tables where database='test_sync_database'" ).strip() == str(number_of_tables) From babe00003620ca34f228009d919d5613db867dee Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 8 Mar 2024 16:43:10 +0000 Subject: [PATCH 076/332] Automatic style fix --- tests/integration/test_replicated_database/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 4f449f9a296..881659262ac 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1141,7 +1141,9 @@ def test_sync_replica(started_cluster): dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") - assert "2\n" == main_node.query("SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'") + assert "2\n" == main_node.query( + "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" + ) assert dummy_node.query( "SELECT count() FROM system.tables where database='test_sync_database'" From 68360aa522169b1c0955837e93687c6d3a124912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 8 Mar 2024 18:37:03 +0100 Subject: [PATCH 077/332] Clang format --- base/base/itoa.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 608258c6b56..9bd2fcd1837 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -34,7 +34,6 @@ namespace { - template ALWAYS_INLINE inline constexpr T pow10(size_t x) { @@ -117,7 +116,6 @@ QuotientAndRemainder inline split(UnsignedOfSize value) return {quotient, remainder}; } - ALWAYS_INLINE inline char * outDigit(char * p, uint8_t value) { *p = '0' + value; @@ -147,7 +145,6 @@ ALWAYS_INLINE inline char * outTwoDigits(char * p, uint8_t value) return p; } - namespace convert { template @@ -296,7 +293,6 @@ ALWAYS_INLINE inline char * itoa(I i, char * p) } } - const uint64_t max_multiple_of_hundred_that_fits_in_64_bits = 1'00'00'00'00'00'00'00'00'00ull; const int max_multiple_of_hundred_blocks = 9; static_assert(max_multiple_of_hundred_that_fits_in_64_bits % 100 == 0); @@ -411,7 +407,6 @@ ALWAYS_INLINE inline char * writeUIntText(UInt256 _x, char * p) return highest_part_print; } - ALWAYS_INLINE inline char * writeLeadingMinus(char * pos) { *pos = '-'; @@ -451,7 +446,6 @@ ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) } } - char * itoa(UInt8 i, char * p) { return convert::itoa(uint8_t(i), p); From bd530a175301860ba1ed13b5b97c56f12e7b13ce Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 8 Mar 2024 20:21:28 +0000 Subject: [PATCH 078/332] Forgot to enable analyzer --- .../03006_parallel_replicas_cte_explain_syntax_crash.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql index a407fceb1c6..f46817d5e82 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -24,7 +24,7 @@ AS SELECT * FROM numbers(1000000) SETTINGS allow_suspicious_low_cardinality_types = 1; -SET allow_experimental_analyzer = 0; +SET allow_experimental_analyzer = 1; SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; EXPLAIN SYNTAX WITH @@ -41,4 +41,3 @@ WITH ) SELECT count() FROM cte2; --- SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; From 2ce96f48f3c3958ef51c3e620b886d633436bb26 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 8 Mar 2024 21:58:01 +0100 Subject: [PATCH 079/332] Update 02962_system_sync_replica_lightweight_from_modifier.sh --- .../02962_system_sync_replica_lightweight_from_modifier.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh b/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh index f47801abf73..b61be87411d 100755 --- a/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh +++ b/tests/queries/0_stateless/02962_system_sync_replica_lightweight_from_modifier.sh @@ -14,7 +14,7 @@ export REPLICAS_TO_DROP for i in $(seq $TOTAL_REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table_$i" - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key" + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key SETTINGS old_parts_lifetime=1" done function insert_thread() { @@ -35,7 +35,7 @@ function sync_and_drop_replicas() { done for i in $(seq $REPLICAS_TO_DROP); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key" + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table_$i (key UInt64, value UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table', '$i') ORDER BY key SETTINGS old_parts_lifetime=1" done done } @@ -87,4 +87,4 @@ for i in $(seq $TOTAL_REPLICAS); do if [ $i -gt $REPLICAS_TO_DROP ]; then $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table_$i" fi -done \ No newline at end of file +done From 23b55ecbe8521692acab507408fc70665aa16f1b Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Fri, 8 Mar 2024 22:02:22 +0100 Subject: [PATCH 080/332] beautify exception --- src/Functions/array/arrayDotProduct.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 97dc9653bab..3f37e6f609f 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -212,7 +212,7 @@ private: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " - "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + "Supported types: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), type_x->getName()); } From 530efbe8102023f857ec29b1f9c7f2090973f736 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 10 Mar 2024 13:08:50 +0000 Subject: [PATCH 081/332] Beautify exception, pt. II --- src/Functions/array/arrayDistance.cpp | 6 +++--- src/Functions/array/arrayDotProduct.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 8b591e37ff6..69e5e3712dd 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -357,7 +357,7 @@ public: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " - "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + "Supported types: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), common_type->getName()); } @@ -412,7 +412,7 @@ private: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " - "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + "Supported types: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), type_x->getName()); } @@ -437,7 +437,7 @@ private: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " - "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + "Supported types: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), type_y->getName()); } diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index 3f37e6f609f..a9547ca90bb 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -237,7 +237,7 @@ private: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " - "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + "Supported types: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", getName(), type_y->getName()); } From bcd705517a510feffd08e9e27c93c2e09eecd144 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 10 Mar 2024 12:53:58 +0000 Subject: [PATCH 082/332] Remove unnecessary call to convertToFullColumnIfConst() --- src/Functions/array/arrayDistance.cpp | 3 --- src/Functions/array/arrayDotProduct.cpp | 3 --- src/Functions/array/arrayNorm.cpp | 3 +-- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 69e5e3712dd..6ed4bf24f99 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -455,9 +455,6 @@ private: return executeWithLeftArgConst(col_y, col_x, input_rows_count, arguments); } - col_x = col_x->convertToFullColumnIfConst(); - col_y = col_y->convertToFullColumnIfConst(); - const auto & array_x = *assert_cast(col_x.get()); const auto & array_y = *assert_cast(col_y.get()); diff --git a/src/Functions/array/arrayDotProduct.cpp b/src/Functions/array/arrayDotProduct.cpp index a9547ca90bb..783843a89d5 100644 --- a/src/Functions/array/arrayDotProduct.cpp +++ b/src/Functions/array/arrayDotProduct.cpp @@ -255,9 +255,6 @@ private: return executeWithLeftArgConst(col_y, col_x, input_rows_count); } - col_x = col_x->convertToFullColumnIfConst(); - col_y = col_y->convertToFullColumnIfConst(); - const auto & array_x = *assert_cast(col_x.get()); const auto & array_y = *assert_cast(col_y.get()); diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 027a33d094c..e87eff6add1 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -175,8 +175,7 @@ public: } } - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { DataTypePtr type = typeid_cast(arguments[0].type.get())->getNestedType(); ColumnPtr column = arguments[0].column->convertToFullColumnIfConst(); From 7a8d79de234e4f83a4e6421e5de49e5800ff70bc Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Sun, 10 Mar 2024 22:13:49 +0100 Subject: [PATCH 083/332] reload CI due to OOM in build From e562d97ff59739da3ca3650a33644bf9700becd7 Mon Sep 17 00:00:00 2001 From: Nikolay Yankin <211292+kolya7k@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:07:19 +0300 Subject: [PATCH 084/332] Update install.md https://packages.clickhouse.com/tgz/stable/ is multi-paged now and sorted by date asc so we can't get very last version --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 234420de374..3b01434ecc5 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ +LATEST_VERSION=$(curl -s -L https://api.github.com/repos/ClickHouse/ClickHouse/tags | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION From db3638ef21b838636a3bb4a607c83b17374067a5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Mar 2024 13:57:07 +0100 Subject: [PATCH 085/332] Enable processors profiling by default It is only 1 record per processor for each query, while there are way more less columns then in query_log, so it is pretty light weight. Signed-off-by: Azat Khuzhin --- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d70a6cf51c5..3275a3f91ef 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -263,7 +263,7 @@ class IColumn; M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \ M(Float, log_queries_probability, 1., "Log queries with the specified probability.", 0) \ \ - M(Bool, log_processors_profiles, false, "Log Processors profile events.", 0) \ + M(Bool, log_processors_profiles, true, "Log Processors profile events.", 0) \ M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \ \ M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index e680c02671a..144affd1b36 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -93,6 +93,7 @@ static std::map sett {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, + {"log_processors_profiles", false, true, "Enable by default"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, From 3c2514bc77837b103c693890515fb1fd87e1befc Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 12 Mar 2024 18:33:32 +0000 Subject: [PATCH 086/332] Correct small issues --- src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 7 +++++-- src/Storages/System/attachSystemTables.cpp | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index ea2c71b0018..6c3a4245a96 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -66,9 +66,12 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step > 1) { + if (step > 1) + { iota_with_step(pos, static_cast(current_end - pos), curr, step); - } else { + } + else + { iota(pos, static_cast(current_end - pos), curr); } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 6d479bc05d7..14125cbd506 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -119,9 +119,9 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attachNoDescription(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false); - attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true); - attach(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); + attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "numbers"); + attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "numbers"); + attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); From dc9548d31b98d6c371d6b84c70bf98ad850bf475 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 13 Mar 2024 18:57:33 +0800 Subject: [PATCH 087/332] add perf test --- tests/performance/array_join.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index ca280ce28ad..b3fc24d3166 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -8,4 +8,6 @@ SELECT count() FROM (SELECT [number] a, [number * 2] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 + + with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) From cf639bab44807e6403f9392c21628a1f86558ef3 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 13 Mar 2024 13:37:26 +0000 Subject: [PATCH 088/332] Fix attach bug --- src/Storages/System/attachSystemTables.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 14125cbd506..3fa55fe28c5 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -119,8 +119,8 @@ namespace DB void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { attachNoDescription(context, system_database, "one", "This table contains a single row with a single dummy UInt8 column containing the value 0. Used when the table is not specified explicitly, for example in queries like `SELECT 1`."); - attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "numbers"); - attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "numbers"); + attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); + attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); From 83019102f3bac7bccfdd25fbbc264bcc1468dc00 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 13 Mar 2024 17:43:48 +0000 Subject: [PATCH 089/332] Correct fast test ande fix perfomance issue --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 25 ++++++++++++++----- .../02992_all_columns_should_have_comment.sql | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 6c3a4245a96..52b5eeb2ca3 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -66,13 +66,13 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step > 1) + if (step == 1) { - iota_with_step(pos, static_cast(current_end - pos), curr, step); + iota(pos, static_cast(current_end - pos), curr); } else { - iota(pos, static_cast(current_end - pos), curr); + iota_with_step(pos, static_cast(current_end - pos), curr, step); } next += chunk_step; @@ -286,7 +286,14 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - iota_with_step(pos, static_cast(size), start_value_64, step); + if (step == 1) + { + iota(pos, static_cast(size), start_value_64); + } + else + { + iota_with_step(pos, static_cast(size), start_value_64, step); + } pos += size; } }; @@ -295,9 +302,15 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - iota_with_step(pos, static_cast(need), start_value, step); + if (step == 1) + { + iota(pos, static_cast(need), start_value); + } + else + { + iota_with_step(pos, static_cast(need), start_value, step); + } pos += need; - provided += need; cursor.offset_in_range += need; } diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql index b984c12e285..b90faeac359 100644 --- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql +++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql @@ -1,4 +1,4 @@ SYSTEM FLUSH LOGS; SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || ' should have a comment' FROM system.columns -WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one')) AND (default_kind != 'ALIAS'); +WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series')) AND (default_kind != 'ALIAS'); From d2b8afb98b39067d057ce0159d24d5879d284e44 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 14 Mar 2024 18:26:13 +0800 Subject: [PATCH 090/332] add benchmarks for column::insertmanyfrom --- src/Columns/CMakeLists.txt | 4 + src/Columns/benchmarks/CMakeLists.txt | 4 + .../benchmark_column_insert_many_from.cpp | 102 ++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 src/Columns/benchmarks/CMakeLists.txt create mode 100644 src/Columns/benchmarks/benchmark_column_insert_many_from.cpp diff --git a/src/Columns/CMakeLists.txt b/src/Columns/CMakeLists.txt index f676f415eea..1febe4f71d7 100644 --- a/src/Columns/CMakeLists.txt +++ b/src/Columns/CMakeLists.txt @@ -1,3 +1,7 @@ if (ENABLE_EXAMPLES) add_subdirectory (examples) endif () + +if (ENABLE_BENCHMARKS) + add_subdirectory(benchmarks) +endif() diff --git a/src/Columns/benchmarks/CMakeLists.txt b/src/Columns/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..47f5dfe4c59 --- /dev/null +++ b/src/Columns/benchmarks/CMakeLists.txt @@ -0,0 +1,4 @@ +clickhouse_add_executable(column_insert_many_from benchmark_column_insert_many_from.cpp) +target_link_libraries (column_insert_many_from PRIVATE + ch_contrib::gbenchmark_all + dbms) \ No newline at end of file diff --git a/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp new file mode 100644 index 00000000000..325cf5559cd --- /dev/null +++ b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +static constexpr size_t ROWS = 65536; + +static ColumnPtr mockColumn(const DataTypePtr & type, size_t rows) +{ + const auto * type_array = typeid_cast(type.get()); + if (type_array) + { + auto data_col = mockColumn(type_array->getNestedType(), rows); + auto offset_col = ColumnArray::ColumnOffsets::create(rows); + auto & offsets = offset_col->getData(); + for (size_t i = 0; i < data_col->size(); ++i) + offsets[i] = offsets[i - 1] + (rand() % 10); + auto new_data_col = data_col->replicate(offsets); + + return ColumnArray::create(new_data_col, std::move(offset_col)); + } + + auto type_not_nullable = removeNullable(type); + auto column = type->createColumn(); + for (size_t i = 0; i < rows; ++i) + { + if (i % 100) + column->insertDefault(); + else if (isInt(type_not_nullable)) + column->insert(i); + else if (isFloat(type_not_nullable)) + { + double d = i * 1.0; + column->insert(d); + } + else if (isString(type_not_nullable)) + { + String s = "helloworld"; + column->insert(s); + } + else + column->insertDefault(); + } + return std::move(column); +} + + +static NO_INLINE void insertManyFrom(IColumn & dst, const IColumn & src) +{ + size_t size = src.size(); + dst.insertManyFrom(src, size / 2, size); +} + + +template +static void BM_insertManyFrom(benchmark::State & state) +{ + auto type = DataTypeFactory::instance().get(str_type); + auto src = mockColumn(type, ROWS); + + for (auto _ : state) + { + state.PauseTiming(); + auto dst = type->createColumn(); + dst->reserve(ROWS); + state.ResumeTiming(); + + insertManyFrom(*dst, *src); + benchmark::DoNotOptimize(dst); + } +} + +static const String type_int64 = "Int64"; +static const String type_nullable_int64 = "Nullable(Int64)"; +static const String type_string = "String"; +static const String type_nullable_string = "Nullable(String)"; +static const String type_decimal = "Decimal128(3)"; +static const String type_nullable_decimal = "Nullable(Decimal128(3))"; + +static const String type_array_int64 = "Array(Int64)"; +static const String type_array_nullable_int64 = "Array(Nullable(Int64))"; +static const String type_array_string = "Array(String)"; +static const String type_array_nullable_string = "Array(Nullable(String))"; + +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_decimal); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_nullable_decimal); + +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_nullable_int64); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_string); +BENCHMARK_TEMPLATE(BM_insertManyFrom, type_array_nullable_string); From 1ac247ed82e583a47c30ee172a856c272f25d00d Mon Sep 17 00:00:00 2001 From: Alexander Sapin Date: Thu, 14 Mar 2024 12:13:33 +0100 Subject: [PATCH 091/332] Tiny improvement for replication.lib --- tests/queries/0_stateless/replication.lib | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib index 143332d9974..2e21f351d2a 100755 --- a/tests/queries/0_stateless/replication.lib +++ b/tests/queries/0_stateless/replication.lib @@ -26,12 +26,15 @@ function try_sync_replicas() done done - i=0 for t in "${tables_arr[@]}" do # Do not start new merges (it can make SYNC a bit faster) $CLICKHOUSE_CLIENT -q "ALTER TABLE $t MODIFY SETTING max_replicated_merges_in_queue=0" + done + i=0 + for t in "${tables_arr[@]}" + do $CLICKHOUSE_CLIENT --receive_timeout $time_left -q "SYSTEM SYNC REPLICA $t STRICT" || ($CLICKHOUSE_CLIENT -q \ "select 'sync failed, queue:', * from system.replication_queue where database=currentDatabase() and table='$t' order by database, table, node_name" && exit 1) & pids[${i}]=$! From f63e1b48e63443aa63acd53962b99c7d645a9507 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Mar 2024 11:21:49 +0000 Subject: [PATCH 092/332] Temporarily increase build limit with sanitizers --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ffb4789dc9..7a07cf2b436 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # -fsanitize=memory and address are too heavy if (SANITIZE) - set (RLIMIT_DATA 10000000000) # 10G + set (RLIMIT_DATA 15000000000) # 10G endif() set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) From a0044f3aa4d5c297229be6f6829f8d7b5e7d75c5 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 14 Mar 2024 11:44:35 +0000 Subject: [PATCH 093/332] Fix ATTACH query with external ON CLUSTER --- src/Interpreters/InterpreterCreateQuery.cpp | 7 +++++-- tests/integration/test_external_cluster/test.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index edd7452c130..fb421ee0a5e 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1087,8 +1087,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { - auto database = DatabaseCatalog::instance().getDatabase(database_name); - if (database->shouldReplicateQuery(getContext(), query_ptr)) + auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); @@ -1099,6 +1099,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!create.cluster.empty()) return executeQueryOnCluster(create); + if (!database) + throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(database_name)); + /// For short syntax of ATTACH query we have to lock table name here, before reading metadata /// and hold it until table is attached if (likely(need_ddl_guard)) diff --git a/tests/integration/test_external_cluster/test.py b/tests/integration/test_external_cluster/test.py index 2ed8ada3df4..6fa9cd16dbc 100644 --- a/tests/integration/test_external_cluster/test.py +++ b/tests/integration/test_external_cluster/test.py @@ -46,6 +46,12 @@ def test_ddl(started_cluster): control_node.query( "ALTER TABLE test_db.test_table ON CLUSTER 'external' add column data String" ) + control_node.query("DETACH TABLE test_db.test_table ON CLUSTER 'external'") + + expected = "" + assert_create_query(data_node, "test_db", "test_table", expected) + + control_node.query("ATTACH TABLE test_db.test_table ON CLUSTER 'external'") expected = "CREATE TABLE test_db.test_table (`id` Int64, `data` String) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192" assert_create_query(data_node, "test_db", "test_table", expected) From b43d1c4eb5c2cb661f3bb8aeb65b4549b8fea6bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Mar 2024 13:13:33 +0100 Subject: [PATCH 094/332] Fix macro --- base/base/itoa.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/itoa.h b/base/base/itoa.h index e69ce0ef17d..3461d679d43 100644 --- a/base/base/itoa.h +++ b/base/base/itoa.h @@ -26,5 +26,5 @@ INSTANTIATION(unsigned long) INSTANTIATION(long) #endif -#undef FOR_MISSING_INTEGER_TYPES +#undef FOR_INTEGER_TYPES #undef INSTANTIATION From bb5e8f52a3fad759007ea6b7684f770f9646c103 Mon Sep 17 00:00:00 2001 From: divanik Date: Thu, 14 Mar 2024 14:27:41 +0000 Subject: [PATCH 095/332] Kek --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a07cf2b436..4799e055698 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,19 +54,19 @@ endif () # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # set DATA (since RSS does not work since 2.6.x+) to 5G - set (RLIMIT_DATA 5000000000) - # set VIRT (RLIMIT_AS) to 10G (DATA*10) - set (RLIMIT_AS 10000000000) - # set CPU time limit to 1000 seconds - set (RLIMIT_CPU 1000) + # # set DATA (since RSS does not work since 2.6.x+) to 5G + # set (RLIMIT_DATA 5000000000) + # # set VIRT (RLIMIT_AS) to 10G (DATA*10) + # set (RLIMIT_AS 10000000000) + # # set CPU time limit to 1000 seconds + # set (RLIMIT_CPU 1000) - # -fsanitize=memory and address are too heavy - if (SANITIZE) - set (RLIMIT_DATA 15000000000) # 10G - endif() + # # -fsanitize=memory and address are too heavy + # if (SANITIZE) + # set (RLIMIT_DATA 15000000000) # 10G + # endif() - set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) + # set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From cd912074a5a94dc80daf4104942e0a9d3ed26e45 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 14 Mar 2024 18:29:07 +0100 Subject: [PATCH 096/332] Fixup --- src/DataTypes/ObjectUtils.cpp | 14 ++++++-------- src/DataTypes/ObjectUtils.h | 4 +--- .../ClusterProxy/SelectStreamFactory.cpp | 19 ++++++++++++------- .../ClusterProxy/SelectStreamFactory.h | 7 +++++-- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++--- .../01455_opentelemetry_distributed.reference | 4 ++-- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index ccfa0a28f13..cdd95da6c00 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -959,24 +959,22 @@ void replaceMissedSubcolumnsByConstants( /// @expected_columns and @available_columns contain descriptions /// of extended Object columns. -MissingObjectList replaceMissedSubcolumnsByConstants( +bool replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, const ContextPtr & context [[maybe_unused]]) { - MissingObjectList missed_list; + bool has_missing_objects = false; NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns); if (missed_names_types.empty()) - return missed_list; + return has_missing_objects; auto * query_node = query->as(); if (!query_node) - return missed_list; - - missed_list.reserve(missed_names_types.size()); + return has_missing_objects; auto table_expression = extractLeftTableExpression(query_node->getJoinTree()); @@ -987,12 +985,12 @@ MissingObjectList replaceMissedSubcolumnsByConstants( constant->setAlias(table_expression->getAlias() + "." + name); column_name_to_node[name] = buildCastFunction(constant, type, context); - missed_list.push_back({ constant->getValueStringRepresentation() + "_" + constant->getResultType()->getName(), table_expression->getAlias() + "." + name }); + has_missing_objects = true; } replaceColumns(query, table_expression, column_name_to_node); - return missed_list; + return has_missing_objects; } Field FieldVisitorReplaceScalars::operator()(const Array & x) const diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 6ef19baf5ae..7b171056f06 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -100,9 +100,7 @@ void replaceMissedSubcolumnsByConstants( const ColumnsDescription & available_columns, ASTPtr query); -using MissingObjectList = std::vector>; - -MissingObjectList replaceMissedSubcolumnsByConstants( +bool replaceMissedSubcolumnsByConstants( const ColumnsDescription & expected_columns, const ColumnsDescription & available_columns, QueryTreeNodePtr & query, diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 4fccd83c8c0..ab301e01d0a 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -130,6 +130,7 @@ void SelectStreamFactory::createForShard( createForShardImpl( shard_info, query_ast, + {}, main_table, table_func_ptr, std::move(context), @@ -143,6 +144,7 @@ void SelectStreamFactory::createForShard( void SelectStreamFactory::createForShardImpl( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, + const QueryTreeNodePtr & query_tree, const StorageID & main_table, const ASTPtr & table_func_ptr, ContextPtr context, @@ -151,13 +153,13 @@ void SelectStreamFactory::createForShardImpl( UInt32 shard_count, bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator, - MissingObjectList missed_list) + bool has_missing_objects) { auto emplace_local_stream = [&]() { Block shard_header; if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); else shard_header = header; @@ -169,15 +171,16 @@ void SelectStreamFactory::createForShardImpl( { Block shard_header; if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_ast, context, SelectQueryOptions(processed_stage).analyze()); + shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); else shard_header = header; remote_shards.emplace_back(Shard{ .query = query_ast, + .query_tree = query_tree, .main_table = main_table, .header = shard_header, - .missing_object_list = std::move(missed_list), + .has_missing_objects = has_missing_objects, .shard_info = shard_info, .lazy = lazy, .local_delay = local_delay, @@ -300,15 +303,17 @@ void SelectStreamFactory::createForShard( auto it = objects_by_shard.find(shard_info.shard_num); QueryTreeNodePtr modified_query = query_tree; - MissingObjectList missed_list; + + bool has_missing_objects = false; if (it != objects_by_shard.end()) - missed_list = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); + has_missing_objects = replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, modified_query, context); auto query_ast = queryNodeToDistributedSelectQuery(modified_query); createForShardImpl( shard_info, query_ast, + modified_query, main_table, table_func_ptr, std::move(context), @@ -317,7 +322,7 @@ void SelectStreamFactory::createForShard( shard_count, parallel_replicas_enabled, std::move(shard_filter_generator), - std::move(missed_list)); + has_missing_objects); } diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 61694830b3d..760281284fd 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -54,11 +54,13 @@ public: { /// Query and header may be changed depending on shard. ASTPtr query; + QueryTreeNodePtr query_tree; + /// Used to check the table existence on remote node StorageID main_table; Block header; - MissingObjectList missing_object_list; + bool has_missing_objects = false; Cluster::ShardInfo shard_info; @@ -110,6 +112,7 @@ private: void createForShardImpl( const Cluster::ShardInfo & shard_info, const ASTPtr & query_ast, + const QueryTreeNodePtr & query_tree, const StorageID & main_table, const ASTPtr & table_func_ptr, ContextPtr context, @@ -118,7 +121,7 @@ private: UInt32 shard_count, bool parallel_replicas_enabled, AdditionalShardFilterGenerator shard_filter_generator, - MissingObjectList missed_list = {}); + bool has_missing_objects = false); }; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 72848a37f6e..8c455883ab2 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -218,7 +218,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream }; pipes.emplace_back(createDelayedPipe(shard.header, lazily_create_stream, add_totals, add_extremes)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFactory::Shard & shard) @@ -299,7 +299,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } } else @@ -328,7 +328,7 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact pipes.emplace_back( createRemoteSourcePipe(remote_query_executor, add_agg_info, add_totals, add_extremes, async_read, async_query_sending)); - addConvertingActions(pipes.back(), output_stream->header, !shard.missing_object_list.empty()); + addConvertingActions(pipes.back(), output_stream->header, shard.has_missing_objects); } } diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index 2920b387aa2..b04a3a5ea82 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,9 +1,9 @@ ===http=== {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"2"} From 6f1b835a4e77df36e322ca16ee01b2c52f60edbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Mar 2024 20:46:54 +0100 Subject: [PATCH 097/332] Fix comment --- base/base/itoa.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 9bd2fcd1837..fd8fd8de025 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -1,6 +1,6 @@ // Based on https://github.com/amdn/itoa and combined with our optimizations // -//=== itoa.h - Fast integer to ascii conversion --*- C++ -*-// +//=== itoa.cpp - Fast integer to ascii conversion --*- C++ -*-// // // The MIT License (MIT) // Copyright (c) 2016 Arturo Martin-de-Nicolas From 25c56ac0a6c291b109bde878b1c3a85c2b1d91ca Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Thu, 14 Mar 2024 18:30:18 -0700 Subject: [PATCH 098/332] [Docs] Specify that MySQL table engine is not supported in Cloud --- docs/en/engines/table-engines/integrations/mysql.md | 4 ++++ utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 87d59a9394b..978f1589e11 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -4,8 +4,12 @@ sidebar_position: 138 sidebar_label: MySQL --- +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + # MySQL + + The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server. ## Creating a Table {#creating-a-table} diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index cbd53e5dfce..e81ef772489 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -161,6 +161,7 @@ ClickHouseVapor ClickVisual ClickableSquare CloudDetails +CloudNotSupportedBadge CodeBlock CodeLLDB Codecs From 22e911dbe42219eb0cfb4fd403c4c520db1cd112 Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Thu, 14 Mar 2024 18:56:28 -0700 Subject: [PATCH 099/332] [Docs] Specify that MySQL database engine is not supported in Cloud --- docs/en/engines/database-engines/mysql.md | 6 +++++- docs/en/engines/table-engines/integrations/mysql.md | 6 +++--- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index 20434ad124e..ac0ea5e5af1 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -4,7 +4,11 @@ sidebar_position: 50 sidebar_label: MySQL --- -# MySQL +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; + +# MySQL Database Engine + + Allows to connect to databases on a remote MySQL server and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and MySQL. diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 978f1589e11..e9d0a43242b 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -4,11 +4,11 @@ sidebar_position: 138 sidebar_label: MySQL --- -import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; +import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge'; -# MySQL +# MySQL Table Engine - + The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server. diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index e81ef772489..f709d9b3546 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -160,6 +160,7 @@ ClickHouseNIO ClickHouseVapor ClickVisual ClickableSquare +CloudAvailableBadge CloudDetails CloudNotSupportedBadge CodeBlock From e5f0079fc66b06317fce92d0736c32a575f8aade Mon Sep 17 00:00:00 2001 From: peter279k Date: Fri, 15 Mar 2024 10:27:10 +0800 Subject: [PATCH 100/332] Correct the GEO datasets information --- docs/en/getting-started/example-datasets/cell-towers.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/getting-started/example-datasets/cell-towers.md b/docs/en/getting-started/example-datasets/cell-towers.md index 090de1b32fd..94fa6998f5d 100644 --- a/docs/en/getting-started/example-datasets/cell-towers.md +++ b/docs/en/getting-started/example-datasets/cell-towers.md @@ -75,14 +75,14 @@ This is the output of `DESCRIBE`. Down further in this guide the field type cho -1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (729 MB). +1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (686 MB). 2. Validate the integrity (optional step): ```bash md5sum cell_towers.csv.xz ``` ```response -8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz +8a797f7bdb55faba93f6cbc37d47b037 cell_towers.csv.xz ``` 3. Decompress it with the following command: @@ -132,7 +132,7 @@ SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC ┌─radio─┬────────c─┐ │ UMTS │ 20686487 │ │ LTE │ 12101148 │ -│ GSM │ 9931312 │ +│ GSM │ 9931304 │ │ CDMA │ 556344 │ │ NR │ 867 │ └───────┴──────────┘ From f55e4f8f289e135ab65ca75fe6bb354b8cf00988 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 14 Mar 2024 09:51:02 +0100 Subject: [PATCH 101/332] Add test. --- .../test_dictionaries_replace/__init__.py | 0 .../configs/allow_database_types.xml | 7 + .../configs/no_lazy_load.xml | 3 + .../configs/remote_servers.xml | 16 +++ .../test_dictionaries_replace/test.py | 136 ++++++++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 tests/integration/test_dictionaries_replace/__init__.py create mode 100644 tests/integration/test_dictionaries_replace/configs/allow_database_types.xml create mode 100644 tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml create mode 100644 tests/integration/test_dictionaries_replace/configs/remote_servers.xml create mode 100644 tests/integration/test_dictionaries_replace/test.py diff --git a/tests/integration/test_dictionaries_replace/__init__.py b/tests/integration/test_dictionaries_replace/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml b/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml new file mode 100644 index 00000000000..0434df06457 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/allow_database_types.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml b/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml new file mode 100644 index 00000000000..aaae3e0c4c1 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/no_lazy_load.xml @@ -0,0 +1,3 @@ + + 0 + diff --git a/tests/integration/test_dictionaries_replace/configs/remote_servers.xml b/tests/integration/test_dictionaries_replace/configs/remote_servers.xml new file mode 100644 index 00000000000..6362032973f --- /dev/null +++ b/tests/integration/test_dictionaries_replace/configs/remote_servers.xml @@ -0,0 +1,16 @@ + + + + + + node_ll + 9000 + + + node_no_ll + 9000 + + + + + diff --git a/tests/integration/test_dictionaries_replace/test.py b/tests/integration/test_dictionaries_replace/test.py new file mode 100644 index 00000000000..bf406f46cb1 --- /dev/null +++ b/tests/integration/test_dictionaries_replace/test.py @@ -0,0 +1,136 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) + +node_ll = cluster.add_instance( + "node_ll", + main_configs=[ + "configs/remote_servers.xml", + ], + user_configs=[ + "configs/allow_database_types.xml", + ], + macros={"replica": "node_ll", "shard": "shard"}, + with_zookeeper=True, +) + +node_no_ll = cluster.add_instance( + "node_no_ll", + main_configs=[ + "configs/no_lazy_load.xml", + "configs/remote_servers.xml", + ], + user_configs=[ + "configs/allow_database_types.xml", + ], + macros={"replica": "node_no_ll", "shard": "shard"}, + with_zookeeper=True, +) + +instances = [node_ll, node_no_ll] + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + # "sleep(1)" is here to make loading of dictionaries a bit slower for this test. + instances[0].query( + "CREATE VIEW src ON CLUSTER 'cluster' AS SELECT number AS key, number * number + 1 + sleep(1) AS value FROM numbers(10)" + ) + + # "dict_get_user" can only call dictGet(), nothing more. + instances[0].query("CREATE USER dictget_user ON CLUSTER 'cluster'") + instances[0].query( + "GRANT dictGet ON atomicdb.dict TO dictget_user ON CLUSTER 'cluster'" + ) + instances[0].query( + "GRANT dictGet ON repldb.dict TO dictget_user ON CLUSTER 'cluster'" + ) + + instances[0].query("CREATE DATABASE atomicdb ON CLUSTER 'cluster'") + instances[0].query( + "CREATE DATABASE repldb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + yield cluster + + finally: + instances[0].query("DROP TABLE IF EXISTS src ON CLUSTER 'cluster'") + instances[0].query("DROP USER IF EXISTS dictget_user ON CLUSTER 'cluster'") + instances[0].query("DROP DATABASE IF EXISTS atomicdb ON CLUSTER 'cluster'") + instances[0].query("DROP DATABASE IF EXISTS repldb ON CLUSTER 'cluster'") + + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + try: + yield + finally: + instances[0].query("DROP DICTIONARY IF EXISTS dict ON CLUSTER 'cluster'") + + +def get_status(instance, dictionary_name): + return instance.query( + "SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'" + ).rstrip("\n") + + +@pytest.mark.parametrize( + "database, instance_to_create_dictionary, instances_to_check", + [ + ("atomicdb", node_ll, [node_ll]), + ("atomicdb", node_no_ll, [node_no_ll]), + ("repldb", node_ll, [node_ll, node_no_ll]), + ("repldb", node_no_ll, [node_ll, node_no_ll]), + ], +) +def test_create_or_replace(database, instance_to_create_dictionary, instances_to_check): + num_steps = 2 + dict_uuids = {} + for step in range(0, num_steps): + create_dictionary_query = f"CREATE OR REPLACE DICTIONARY {database}.dict (key Int64, value Int64) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'src' PASSWORD '' DB 'default')) LIFETIME(0) LAYOUT(FLAT())" + instance_to_create_dictionary.query(create_dictionary_query) + + for instance in instances_to_check: + if instance != instance_to_create_dictionary: + instance.query(f"SYSTEM SYNC DATABASE REPLICA {database}") + + dict_uuid = instance.query( + f"SELECT uuid FROM system.tables WHERE database='{database}' AND table='dict'" + ).strip("\n") + + dict_db, dict_name, dict_status = ( + instance.query( + f"SELECT database, name, status FROM system.dictionaries WHERE uuid='{dict_uuid}'" + ) + .strip("\n") + .split("\t") + ) + assert dict_db == database + assert dict_name == "dict" + + # "uuid" must be the same for all the dictionaries created at the same "step" and different for the dictionaries created at different steps. + if step in dict_uuids: + assert dict_uuids[step] == dict_uuid + dict_uuids[step] = dict_uuid + assert dict_uuid not in [ + dict_uuids[prev_step] for prev_step in range(0, step) + ] + + expected_dict_status = ( + ["NOT_LOADED"] if instance == node_ll else ["LOADING", "LOADED"] + ) + assert dict_status in expected_dict_status + + for instance in instances_to_check: + select_query = f"SELECT arrayJoin([0, 5, 7, 11]) as key, dictGet({database}.dict, 'value', key)" + expected_result = TSV([[0, 1], [5, 26], [7, 50], [11, 0]]) + assert instance.query(select_query) == expected_result + assert instance.query(select_query, user="dictget_user") == expected_result From 00533f3df634c3a96e78fa0732c9375f257ffb5b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 15 Mar 2024 19:43:44 +0800 Subject: [PATCH 102/332] revert opts in column array --- src/Columns/ColumnArray.cpp | 232 +++--------------------------------- src/Columns/ColumnArray.h | 14 +-- 2 files changed, 19 insertions(+), 227 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 0214375122f..7b268b80116 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -31,7 +31,6 @@ namespace ErrorCodes extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; extern const int LOGICAL_ERROR; extern const int TOO_LARGE_ARRAY_SIZE; - extern const int ILLEGAL_COLUMN; } /** Obtaining array as Field can be slow for large arrays and consume vast amount of memory. @@ -43,34 +42,29 @@ namespace ErrorCodes static constexpr size_t max_array_size_as_field = 1000000; -ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - if (check_offsets) + const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); + + if (!offsets_concrete) + throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); + + if (!offsets_concrete->empty() && data && !data->empty()) { - const ColumnOffsets * offsets_concrete = typeid_cast(offsets.get()); + Offset last_offset = offsets_concrete->getData().back(); - if (!offsets_concrete) - throw Exception(ErrorCodes::LOGICAL_ERROR, "offsets_column must be a ColumnUInt64"); - - if (!offsets_concrete->empty() && data && !data->empty()) - { - Offset last_offset = offsets_concrete->getData().back(); - - /// This will also prevent possible overflow in offset. - if (data->size() != last_offset) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", - data->size(), - last_offset); - } - - /** NOTE - * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). - * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. - */ + /// This will also prevent possible overflow in offset. + if (data->size() != last_offset) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "offsets_column has data inconsistent with nested_column. Data size: {}, last offset: {}", + data->size(), last_offset); } + + /** NOTE + * Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate). + * But in most cases, arrays with constant value are unexpected and code will work wrong. Use with caution. + */ } ColumnArray::ColumnArray(MutableColumnPtr && nested_column) @@ -353,196 +347,6 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n) getOffsets().push_back(getOffsets().back() + size); } -template -void ColumnArray::insertManyFromNumber(const ColumnArray & src, size_t position, size_t length) -{ - using ColVecType = ColumnVectorOrDecimal; - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - - const typename ColVecType::Container & src_data = typeid_cast(src.getData()).getData(); - typename ColVecType::Container & data_ref = typeid_cast(getData()).getData(); - size_t old_size = data_ref.size(); - size_t new_size = old_size + src_size * length; - data_ref.resize(new_size); - for (size_t i = 0, offset = old_size; i < length; ++i, offset += src_size) - memcpy(&data_ref[offset], &src_data[src_offset], src_size * sizeof(T)); -} - -void ColumnArray::insertManyFromConst(const ColumnConst & src, size_t /*position*/, size_t length) -{ - const ColumnArray * src_array = typeid_cast(&src.getDataColumn()); - if (!src_array) - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, - "Cannot insert from const column of type {} to column of type {}", - src.getDataColumn().getName(), - getName()); - - insertManyFromImpl(*src_array, 0, length, true); -} - -void ColumnArray::insertManyFromString(const ColumnArray & src, size_t position, size_t length) -{ - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - - const auto & src_string = typeid_cast(src.getData()); - const auto & src_chars = src_string.getChars(); - const auto & src_string_offsets = src_string.getOffsets(); - auto & dst_string = typeid_cast(getData()); - auto & dst_chars = dst_string.getChars(); - auto & dst_string_offsets = dst_string.getOffsets(); - - /// Each row may have multiple strings, copy them to dst_chars and update dst_offsets - size_t old_size = dst_string_offsets.size(); - size_t new_size = old_size + src_size * length; - dst_string_offsets.resize(new_size); - size_t dst_string_offset = dst_chars.size(); - for (size_t i = 0; i < length; ++i) - { - for (size_t j = 0; j < src_size; ++j) - { - size_t nested_offset = src_string_offsets[src_offset + j - 1]; - size_t nested_length = src_string_offsets[src_offset + j] - nested_offset; - - dst_string_offset += nested_length; - dst_string_offsets[old_size + i * src_size + j] = dst_string_offset; - } - } - - size_t chars_to_copy = src_string_offsets[src_offset + src_size - 1] - src_string_offsets[src_offset - 1]; - dst_chars.resize(dst_chars.size() + chars_to_copy * length); - for (size_t dst_offset = old_size; dst_offset < new_size; dst_offset += src_size) - memcpy(&dst_chars[dst_string_offsets[dst_offset - 1]], &src_chars[src_string_offsets[src_offset - 1]], chars_to_copy); -} - -void ColumnArray::insertManyFromTuple(const ColumnArray & src, size_t position, size_t length) -{ - ColumnTuple & tuple = assert_cast(getData()); - const ColumnTuple & src_tuple = assert_cast(src.getData()); - - /// Make temporary arrays for each components of Tuple. In the same way as for Nullable. - size_t tuple_size = tuple.tupleSize(); - size_t src_tuple_size = src_tuple.tupleSize(); - if (tuple_size == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty tuple"); - if (tuple_size != src_tuple_size) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Nested tuple size mismatch: {} vs {}", tuple_size, src_tuple_size); - - Columns tuple_columns(tuple_size); - for (size_t i = 0; i < tuple_size; ++i) - { - auto array_of_element = ColumnArray(tuple.getColumn(i).assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_element = ColumnArray(src_tuple.getColumn(i).assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_element.insertManyFromImpl(src_array_of_element, position, length, false); - tuple_columns[i] = array_of_element.getDataPtr(); - } - getDataPtr() = ColumnTuple::create(std::move(tuple_columns)); -} - -void ColumnArray::insertManyFromNullable(const ColumnArray & src, size_t position, size_t length) -{ - ColumnNullable & nullable = assert_cast(getData()); - const ColumnNullable & src_nullable = assert_cast(src.getData()); - - /// Process nested column without updating array offsets - auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_nested = ColumnArray(src_nullable.getNestedColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_nested.insertManyFromImpl(src_array_of_nested, position, length, false); - - /// Process null map column without updating array offsets - auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable(), false); - auto src_array_of_null_map = ColumnArray(src_nullable.getNullMapColumnPtr()->assumeMutable(), src.getOffsetsPtr()->assumeMutable()); - array_of_null_map.insertManyFromImpl(src_array_of_null_map, position, length, false); - - /// Update array data - getDataPtr() = ColumnNullable::create(array_of_nested.getDataPtr(), array_of_null_map.getDataPtr()); -} - -void ColumnArray::insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length) -{ - size_t src_size = src.sizeAt(position); - size_t src_offset = src.offsetAt(position); - const auto & src_data = src.getData(); - size_t new_size = data->size() + src_size * length; - data->reserve(new_size); - for (size_t i = 0; i < length; ++i) - data->insertRangeFrom(src_data, src_offset, src_size); -} - -void ColumnArray::insertManyFrom(const IColumn & src_, size_t position, size_t length) -{ - const ColumnConst * src_const = typeid_cast(&src_); - if (src_const) - return insertManyFromConst(*src_const, position, length); - - const ColumnArray * src_array = typeid_cast(&src_); - if (!src_array) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert from column of type {} to column of type {}", src_.getName(), getName()); - - return insertManyFromImpl(*src_array, position, length, true); -} - -void ColumnArray::insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets) -{ - /// First fill offsets if needed - if (update_offsets) - { - size_t src_size = src.sizeAt(position); - auto & offsets_ref = getOffsets(); - size_t old_rows = offsets_ref.size(); - size_t new_rows = old_rows + length; - size_t old_size = offsets_ref.back(); - offsets_ref.resize(new_rows); - for (size_t i = 0, offset = old_size + src_size; i < length; ++i, offset += src_size) - offsets_ref[old_rows + i] = offset; - } - - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast *>(data.get())) - return insertManyFromNumber(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromString(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromNullable(src, position, length); - if (typeid_cast(data.get())) - return insertManyFromTuple(src, position, length); - return insertManyFromGeneric(src, position, length); -} void ColumnArray::insertDefault() { diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 8c4d103e7d0..230d8830265 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -21,7 +21,7 @@ private: friend class COWHelper, ColumnArray>; /** Create an array column with specified values and offsets. */ - ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column, bool check_offsets = true); + ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ explicit ColumnArray(MutableColumnPtr && nested_column); @@ -88,7 +88,6 @@ public: void insert(const Field & x) override; bool tryInsert(const Field & x) override; void insertFrom(const IColumn & src_, size_t n) override; - void insertManyFrom(const IColumn & src, size_t position, size_t length) override; void insertDefault() override; void popBack(size_t n) override; ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; @@ -214,17 +213,6 @@ private: ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; - /// Specializations for insertManyFrom - void insertManyFromConst(const ColumnConst & src, size_t position, size_t length); - void insertManyFromImpl(const ColumnArray & src, size_t position, size_t length, bool update_offsets = true); - - template - void insertManyFromNumber(const ColumnArray & src, size_t position, size_t length); - void insertManyFromString(const ColumnArray & src, size_t position, size_t length); - void insertManyFromTuple(const ColumnArray & src, size_t position, size_t length); - void insertManyFromNullable(const ColumnArray & src, size_t position, size_t length); - void insertManyFromGeneric(const ColumnArray & src, size_t position, size_t length); - int compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint, const Collator * collator=nullptr) const; }; From 7d38789e3053b6912300e9fd48456cd3b3426690 Mon Sep 17 00:00:00 2001 From: divanik Date: Fri, 15 Mar 2024 14:43:01 +0000 Subject: [PATCH 103/332] Make generate_series test smaller --- tests/queries/0_stateless/02970_generate_series.reference | 5 +++-- tests/queries/0_stateless/02970_generate_series.sql | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference index 4e4f556a39b..6e6f3c81587 100644 --- a/tests/queries/0_stateless/02970_generate_series.reference +++ b/tests/queries/0_stateless/02970_generate_series.reference @@ -2,8 +2,9 @@ 1 4 8 -500000001 -50000000 +501 +50 +17928 0 10 13 diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql index a7f89e1bd3f..0844714b3a6 100644 --- a/tests/queries/0_stateless/02970_generate_series.sql +++ b/tests/queries/0_stateless/02970_generate_series.sql @@ -2,8 +2,9 @@ SELECT count() FROM generate_series(5, 4); SELECT count() FROM generate_series(0, 0); SELECT count() FROM generate_series(10, 20, 3); SELECT count() FROM generate_series(7, 77, 10); -SELECT count() FROM generate_series(0, 1000000000, 2); -SELECT count() FROM generate_series(0, 999999999, 20); +SELECT count() FROM generate_series(0, 1000, 2); +SELECT count() FROM generate_series(0, 999, 20); +SELECT sum(generate_series) FROM generate_series(4, 1008, 4) WHERE generate_series % 7 = 1; SELECT * FROM generate_series(5, 4); SELECT * FROM generate_series(0, 0); From 7c49d5bbf673b5ca855bae79609c6a2ea5bc2079 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 15 Mar 2024 15:35:31 +0000 Subject: [PATCH 104/332] Fix some tests #ci_set_analyzer --- .../ClusterProxy/SelectStreamFactory.cpp | 8 +------- .../OptimizeShardingKeyRewriteInVisitor.cpp | 8 +++++++- .../QueryPlan/DistributedCreateLocalPlan.cpp | 13 ++++++++----- .../QueryPlan/DistributedCreateLocalPlan.h | 3 ++- src/Processors/QueryPlan/ReadFromRemote.cpp | 6 +++++- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index ab301e01d0a..94a02eacdaa 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -157,14 +157,8 @@ void SelectStreamFactory::createForShardImpl( { auto emplace_local_stream = [&]() { - Block shard_header; - if (context->getSettingsRef().allow_experimental_analyzer) - shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); - else - shard_header = header; - local_plans.emplace_back(createLocalPlan( - query_ast, shard_header, context, processed_stage, shard_info.shard_num, shard_count)); + query_ast, header, context, processed_stage, shard_info.shard_num, shard_count, has_missing_objects)); }; auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0) diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index e0d0fda81cb..0df6fcee229 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -11,6 +11,7 @@ #include "Analyzer/FunctionNode.h" #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/InDepthQueryTreeVisitor.h" +#include "Analyzer/Utils.h" #include "DataTypes/IDataType.h" #include "Interpreters/Context_fwd.h" @@ -170,7 +171,12 @@ public: if (new_tuple.empty()) new_tuple.push_back(tuple.back()); - node = std::make_shared(new_tuple); + + if (new_tuple.size() == tuple.size()) + return; + + arguments[1] = std::make_shared(new_tuple); + rerunFunctionResolve(function_node, getContext()); } } } diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 4d4bc6220e9..c8d230c87d9 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -14,12 +14,14 @@ namespace DB namespace { -void addConvertingActions(QueryPlan & plan, const Block & header) +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) { if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) return; - auto get_converting_dag = [](const Block & block_, const Block & header_) + auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode](const Block & block_, const Block & header_) { /// Convert header structure to expected. /// Also we ignore constants from result and replace it with constants from header. @@ -27,7 +29,7 @@ void addConvertingActions(QueryPlan & plan, const Block & header) return ActionsDAG::makeConvertingActions( block_.getColumnsWithTypeAndName(), header_.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name, + mode, true); }; @@ -44,7 +46,8 @@ std::unique_ptr createLocalPlan( ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t shard_num, - size_t shard_count) + size_t shard_count, + bool has_missing_objects) { checkStackSize(); @@ -74,7 +77,7 @@ std::unique_ptr createLocalPlan( interpreter.buildQueryPlan(*query_plan); } - addConvertingActions(*query_plan, header); + addConvertingActions(*query_plan, header, has_missing_objects); return query_plan; } diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h index 489d2607849..50545d9ae81 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h @@ -18,5 +18,6 @@ std::unique_ptr createLocalPlan( ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t shard_num, - size_t shard_count); + size_t shard_count, + bool has_missing_objects); } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 8c455883ab2..119710d06d8 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,9 @@ static void addConvertingActions(Pipe & pipe, const Block & header, bool use_pos true); }; + if (use_positions_to_match) + pipe.addSimpleTransform([](const Block & stream_header) { return std::make_shared(stream_header); }); + auto convert_actions = std::make_shared(get_converting_dag(pipe.getHeader(), header)); pipe.addSimpleTransform([&](const Block & cur_header, Pipe::StreamType) -> ProcessorPtr { @@ -190,7 +194,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream if (try_results.empty() || local_delay < max_remote_delay) { auto plan = createLocalPlan( - query, header, my_context, my_stage, my_shard.shard_info.shard_num, my_shard_count); + query, header, my_context, my_stage, my_shard.shard_info.shard_num, my_shard_count, my_shard.has_missing_objects); return std::move(*plan->buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(my_context), From b9f6de095d5932c83abe0092c4b7b7e29a64e4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 15 Mar 2024 18:11:35 +0100 Subject: [PATCH 105/332] Disable async_insert_use_adaptive_busy_timeout correctly with compatibility settings --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 4914f97a6fb..5ecaf12647f 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -104,7 +104,7 @@ static std::map sett {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"}, {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."}, {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"}, - {"async_insert_use_adaptive_busy_timeout", true, true, "Use adaptive asynchronous insert timeout"}, + {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"}, {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"}, {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"}, {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, From cfb13d212e5e9093542fef7875fd4df764a8dd7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 15 Mar 2024 18:28:04 +0100 Subject: [PATCH 106/332] Add a test --- .../0_stateless/03011_adaptative_timeout_compatibility.reference | 1 + .../0_stateless/03011_adaptative_timeout_compatibility.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/03011_adaptative_timeout_compatibility.reference create mode 100644 tests/queries/0_stateless/03011_adaptative_timeout_compatibility.sql diff --git a/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.reference b/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.sql b/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.sql new file mode 100644 index 00000000000..cdd2da0f6d8 --- /dev/null +++ b/tests/queries/0_stateless/03011_adaptative_timeout_compatibility.sql @@ -0,0 +1 @@ +SELECT value from system.settings where name = 'async_insert_use_adaptive_busy_timeout' SETTINGS compatibility = '23.12'; From adaf262dc09ff08628d54d5ffd64f7ea5053482d Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Sat, 16 Mar 2024 01:12:59 +0000 Subject: [PATCH 107/332] Fix bug when reading system.parts using UUID (issue 61220). During the predicates pushing down of system.parts table, there is a logic to use that predicates to filter on the storages. This makes sense if the predicate is on the `database`, `table`, `engine`. But it can be problem if the predicate is about `uuid`, because storage UUID is apparently different from parts UUID. Rename the column name from `uuid` to `storage_uuid` fixed this. --- src/Storages/System/StorageSystemPartsBase.cpp | 4 ++-- .../03010_read_system_parts_table_test.reference | 4 ++++ .../0_stateless/03010_read_system_parts_table_test.sql | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03010_read_system_parts_table_test.reference create mode 100644 tests/queries/0_stateless/03010_read_system_parts_table_test.sql diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..1283f2d6ce0 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -188,7 +188,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte block_to_filter.insert(ColumnWithTypeAndName(std::move(table_column_mut), std::make_shared(), "table")); block_to_filter.insert(ColumnWithTypeAndName(std::move(engine_column_mut), std::make_shared(), "engine")); block_to_filter.insert(ColumnWithTypeAndName(std::move(active_column_mut), std::make_shared(), "active")); - block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), "uuid")); + block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), "storage_uuid")); if (rows) { @@ -200,7 +200,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte database_column = block_to_filter.getByName("database").column; table_column = block_to_filter.getByName("table").column; active_column = block_to_filter.getByName("active").column; - storage_uuid_column = block_to_filter.getByName("uuid").column; + storage_uuid_column = block_to_filter.getByName("storage_uuid").column; } class ReadFromSystemPartsBase : public SourceStepWithFilter diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference new file mode 100644 index 00000000000..6847786761e --- /dev/null +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference @@ -0,0 +1,4 @@ +00000000-0000-0000-0000-000000000000 1231_1_1_0 +00000000-0000-0000-0000-000000000000 6666_2_2_0 +users +users diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql new file mode 100644 index 00000000000..6ec7e6212d5 --- /dev/null +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS users; +CREATE TABLE users (uid Int16, name String, age Int16) ENGINE=MergeTree ORDER BY uid PARTITION BY uid; + +INSERT INTO users VALUES (1231, 'John', 33); +INSERT INTO users VALUES (6666, 'Ksenia', 48); + +SELECT uuid, name from system.parts WHERE table = 'users'; + +SELECT table from system.parts WHERE uuid = '00000000-0000-0000-0000-000000000000'; +DROP TABLE IF EXISTS users; From 335f147a3b6e69d7924a2f70449f5b2ee125bac6 Mon Sep 17 00:00:00 2001 From: Nikolay Yankin <211292+kolya7k@users.noreply.github.com> Date: Sat, 16 Mar 2024 16:57:39 +0300 Subject: [PATCH 108/332] Update install.md Alternative URL for latest version --- docs/en/getting-started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 3b01434ecc5..cf95560b311 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -LATEST_VERSION=$(curl -s -L https://api.github.com/repos/ClickHouse/ClickHouse/tags | \ +LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/utils/list-versions/version_date.tsv | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION From 41d8621d975497f7a5b0c1fbe845a6410a2a5341 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 14 Mar 2024 23:50:01 +0100 Subject: [PATCH 109/332] Fix replacing dictionaries: StorageID is now updated immediately for every dictionary during CREATE OR REPLACE even if the dictionary is not loaded yet or loading right now. --- src/Dictionaries/IDictionary.h | 18 +++++++------ .../ExternalDictionariesLoader.cpp | 14 ++++++++++ src/Interpreters/ExternalDictionariesLoader.h | 3 +++ src/Interpreters/ExternalLoader.cpp | 21 ++++++++++++--- src/Interpreters/ExternalLoader.h | 4 +++ src/Storages/StorageDictionary.cpp | 27 ++++++++++--------- 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index bab3e470b79..9770e93669e 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -75,13 +75,6 @@ public: return dictionary_id; } - void updateDictionaryName(const StorageID & new_name) const - { - std::lock_guard lock{mutex}; - assert(new_name.uuid == dictionary_id.uuid && dictionary_id.uuid != UUIDHelpers::Nil); - dictionary_id = new_name; - } - std::string getLoadableName() const final { std::lock_guard lock{mutex}; @@ -452,9 +445,18 @@ public: return sample_block; } + /// Internally called by ExternalDictionariesLoader. + /// In order to update the dictionary ID change its configuration first and then call ExternalDictionariesLoader::reloadConfig(). + void updateDictionaryID(const StorageID & new_dictionary_id) + { + std::lock_guard lock{mutex}; + assert((new_dictionary_id.uuid == dictionary_id.uuid) && (dictionary_id.uuid != UUIDHelpers::Nil)); + dictionary_id = new_dictionary_id; + } + private: mutable std::mutex mutex; - mutable StorageID dictionary_id TSA_GUARDED_BY(mutex); + StorageID dictionary_id TSA_GUARDED_BY(mutex); String dictionary_comment TSA_GUARDED_BY(mutex); }; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 74984de0064..760f94d18c2 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -41,6 +41,20 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, getContext(), created_from_ddl); } +void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const +{ + IDictionary & dict = static_cast(object); + + auto new_dictionary_id = StorageID::fromDictionaryConfig(config, key_in_config); + auto old_dictionary_id = dict.getDictionaryID(); + if ((new_dictionary_id.table_name != old_dictionary_id.table_name) || (new_dictionary_id.database_name != old_dictionary_id.database_name)) + { + /// We can update the dictionary ID without reloading only if it's in the atomic database. + if ((new_dictionary_id.uuid == old_dictionary_id.uuid) && (new_dictionary_id.uuid != UUIDHelpers::Nil)) + dict.updateDictionaryID(new_dictionary_id); + } +} + ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 5f27d695c8b..3c226f998be 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -43,6 +43,9 @@ protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; + void updateObjectFromConfigWithoutReloading( + IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const override; + std::string resolveDictionaryName(const std::string & dictionary_name, const std::string & current_database_name) const; /// Try convert qualified dictionary name to persistent UUID diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 36664cbd06f..ec429054475 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -87,6 +87,8 @@ namespace lock = std::unique_lock(mutex); } }; + + using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -377,7 +379,7 @@ private: } const String type_name; - LoggerPtr log; + const LoggerPtr log; std::mutex mutex; ExternalLoaderConfigSettings settings; @@ -400,9 +402,11 @@ public: LoadingDispatcher( const CreateObjectFunction & create_object_function_, + const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, const String & type_name_, LoggerPtr log_) : create_object(create_object_function_) + , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) , type_name(type_name_) , log(log_) { @@ -457,10 +461,15 @@ public: else { const auto & new_config = new_config_it->second; - bool config_is_same = isSameConfiguration(*info.config->config, info.config->key_in_config, *new_config->config, new_config->key_in_config); + auto previous_config = info.config; info.config = new_config; - if (!config_is_same) + + bool config_changed = !isSameConfiguration(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + if (config_changed) { + if (info.object) + update_object_from_config_without_reloading(*info.object, *new_config->config, new_config->key_in_config); + if (info.triedToLoad()) { /// The object has been tried to load before, so it is currently in use or was in use @@ -1117,7 +1126,10 @@ private: } if (new_object) + { + update_object_from_config_without_reloading(*new_object, *info->config->config, info->config->key_in_config); info->object = new_object; + } info->exception = new_exception; info->error_count = error_count; @@ -1192,6 +1204,7 @@ private: } const CreateObjectFunction create_object; + const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; LoggerPtr log; @@ -1277,6 +1290,8 @@ ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) , loading_dispatcher(std::make_unique( [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, + [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) + { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, type_name_, log_)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index a5d83bdab50..279157b6d51 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -213,6 +213,10 @@ public: protected: virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + /// Updates the object from the configuration without reloading as much as possible. + virtual void updateObjectFromConfigWithoutReloading( + IExternalLoadable & /* object */, const Poco::Util::AbstractConfiguration & /* config */, const String & /* key_in_config */) const {} + private: void checkLoaded(const LoadResult & result, bool check_no_errors) const; void checkLoaded(const LoadResults & results, bool check_no_errors) const; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index f716332dd24..0c6f6a25b9d 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -221,6 +221,10 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) bool move_to_ordinary = old_table_id.uuid != UUIDHelpers::Nil && new_table_id.uuid == UUIDHelpers::Nil; assert(old_table_id.uuid == new_table_id.uuid || move_to_atomic || move_to_ordinary); + /// It's better not to update an associated `IDictionary` directly here because it can be not loaded yet or + /// it can be in the process of loading or reloading right now. + /// The correct way is to update the dictionary's configuration first and then ask ExternalDictionariesLoader to reload our dictionary. + { std::lock_guard lock(dictionary_config_mutex); @@ -232,31 +236,28 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) configuration->remove("dictionary.uuid"); } + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + /// Dictionary is moving between databases of different engines or is renaming inside Ordinary database bool recreate_dictionary = old_table_id.uuid == UUIDHelpers::Nil || new_table_id.uuid == UUIDHelpers::Nil; if (recreate_dictionary) { - /// It's too hard to update both name and uuid, better to reload dictionary with new name + /// For an ordinary database the config repositories of dictionaries are identified by the full name (database name + dictionary name), + /// so we cannot change the dictionary name or the database name on the fly (without extra reloading) and have to recreate the config repository. removeDictionaryConfigurationFromRepository(); auto repository = std::make_unique(*this); - remove_repository_callback = getContext()->getExternalDictionariesLoader().addConfigRepository(std::move(repository)); - /// Dictionary will be reloaded lazily to avoid exceptions in the middle of renaming + remove_repository_callback = external_dictionaries_loader.addConfigRepository(std::move(repository)); + /// Dictionary will be now reloaded lazily. } else { - const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); - auto result = external_dictionaries_loader.getLoadResult(old_table_id.getInternalDictionaryName()); - - if (result.object) - { - const auto dictionary = std::static_pointer_cast(result.object); - dictionary->updateDictionaryName(new_table_id); - } - + /// For an atomic database dictionaries are identified inside the ExternalLoader by UUID, + /// so we can change the dictionary name or the database name on the fly (without extra reloading) because UUID doesn't change. external_dictionaries_loader.reloadConfig(old_table_id.getInternalDictionaryName()); - dictionary_name = new_table_id.getFullNameNotQuoted(); } + + dictionary_name = new_table_id.getFullNameNotQuoted(); } void StorageDictionary::checkAlterIsPossible(const AlterCommands & commands, ContextPtr /* context */) const From 4fe1c9a78f99b915dd4f31cbf37a2c3db6f49e1d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 00:07:53 +0100 Subject: [PATCH 110/332] Get rid of checking the "last modification time" when the configuration is generated to allow ExternalLoader::reloadConfig() work for generated configurations too. --- src/Interpreters/ExternalLoader.cpp | 4 ++-- .../ExternalLoaderDictionaryStorageConfigRepository.cpp | 5 ----- .../ExternalLoaderDictionaryStorageConfigRepository.h | 2 -- src/Interpreters/ExternalLoaderTempConfigRepository.cpp | 8 -------- src/Interpreters/ExternalLoaderTempConfigRepository.h | 2 -- src/Interpreters/ExternalLoaderXMLConfigRepository.cpp | 2 +- src/Interpreters/ExternalLoaderXMLConfigRepository.h | 2 +- src/Interpreters/IExternalLoaderConfigRepository.h | 2 +- src/Storages/StorageDictionary.cpp | 6 ------ src/Storages/StorageDictionary.h | 2 -- 10 files changed, 5 insertions(+), 30 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index ec429054475..3e162b1114f 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -169,7 +169,7 @@ public: private: struct FileInfo { - Poco::Timestamp last_update_time = 0; + std::optional last_update_time; bool in_use = true; // Whether the `FileInfo` should be destroyed because the correspondent file is deleted. Poco::AutoPtr file_contents; // Parsed contents of the file. std::unordered_map objects; @@ -268,7 +268,7 @@ private: // is updated, but in the same second). // The solution to this is probably switching to std::filesystem // -- the work is underway to do so. - if (update_time_from_repository == file_info.last_update_time) + if (update_time_from_repository && (update_time_from_repository == file_info.last_update_time)) { file_info.in_use = true; return false; diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp index 86f5a9ded0a..042992fcd2e 100644 --- a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp @@ -26,11 +26,6 @@ bool ExternalLoaderDictionaryStorageConfigRepository::exists(const std::string & return getName() == loadable_definition_name; } -Poco::Timestamp ExternalLoaderDictionaryStorageConfigRepository::getUpdateTime(const std::string &) -{ - return dictionary_storage.getUpdateTime(); -} - LoadablesConfigurationPtr ExternalLoaderDictionaryStorageConfigRepository::load(const std::string &) { return dictionary_storage.getConfiguration(); diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h index 06d2b0faf75..f357df990e6 100644 --- a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h @@ -19,8 +19,6 @@ public: bool exists(const std::string & loadable_definition_name) override; - Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) override; - LoadablesConfigurationPtr load(const std::string & loadable_definition_name) override; private: diff --git a/src/Interpreters/ExternalLoaderTempConfigRepository.cpp b/src/Interpreters/ExternalLoaderTempConfigRepository.cpp index 10fc61a2ed0..0a8fc170294 100644 --- a/src/Interpreters/ExternalLoaderTempConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderTempConfigRepository.cpp @@ -28,14 +28,6 @@ bool ExternalLoaderTempConfigRepository::exists(const String & path_) } -Poco::Timestamp ExternalLoaderTempConfigRepository::getUpdateTime(const String & path_) -{ - if (!exists(path_)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Loadable {} not found", path_); - return creation_time; -} - - LoadablesConfigurationPtr ExternalLoaderTempConfigRepository::load(const String & path_) { if (!exists(path_)) diff --git a/src/Interpreters/ExternalLoaderTempConfigRepository.h b/src/Interpreters/ExternalLoaderTempConfigRepository.h index 92816dcb7f2..a3c67b3eed6 100644 --- a/src/Interpreters/ExternalLoaderTempConfigRepository.h +++ b/src/Interpreters/ExternalLoaderTempConfigRepository.h @@ -18,14 +18,12 @@ public: std::set getAllLoadablesDefinitionNames() override; bool exists(const String & path) override; - Poco::Timestamp getUpdateTime(const String & path) override; LoadablesConfigurationPtr load(const String & path) override; private: String name; String path; LoadablesConfigurationPtr config; - Poco::Timestamp creation_time; }; } diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp index 8b9584f1242..a15f918f457 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.cpp @@ -24,7 +24,7 @@ ExternalLoaderXMLConfigRepository::ExternalLoaderXMLConfigRepository( { } -Poco::Timestamp ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) +std::optional ExternalLoaderXMLConfigRepository::getUpdateTime(const std::string & definition_entity_name) { return FS::getModificationTimestamp(definition_entity_name); } diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.h b/src/Interpreters/ExternalLoaderXMLConfigRepository.h index 25646d72db3..814817cad6f 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.h +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.h @@ -26,7 +26,7 @@ public: bool exists(const std::string & definition_entity_name) override; /// Return xml-file modification time via stat call - Poco::Timestamp getUpdateTime(const std::string & definition_entity_name) override; + std::optional getUpdateTime(const std::string & definition_entity_name) override; /// May contain definition about several entities (several dictionaries in one .xml file) LoadablesConfigurationPtr load(const std::string & config_file) override; diff --git a/src/Interpreters/IExternalLoaderConfigRepository.h b/src/Interpreters/IExternalLoaderConfigRepository.h index 0d0c8acc01a..6ba4add86e2 100644 --- a/src/Interpreters/IExternalLoaderConfigRepository.h +++ b/src/Interpreters/IExternalLoaderConfigRepository.h @@ -37,7 +37,7 @@ public: virtual bool exists(const std::string & path) = 0; /// Returns entity last update time - virtual Poco::Timestamp getUpdateTime(const std::string & path) = 0; + virtual std::optional getUpdateTime(const std::string & /* path */) { return {}; } /// Load configuration from some concrete source to AbstractConfiguration virtual LoadablesConfigurationPtr load(const std::string & path) = 0; diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 0c6f6a25b9d..ee76522fe48 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -194,12 +194,6 @@ void StorageDictionary::removeDictionaryConfigurationFromRepository() remove_repository_callback.reset(); } -Poco::Timestamp StorageDictionary::getUpdateTime() const -{ - std::lock_guard lock(dictionary_config_mutex); - return update_time; -} - LoadablesConfigurationPtr StorageDictionary::getConfiguration() const { std::lock_guard lock(dictionary_config_mutex); diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 995a0192269..5c6b2f6d8ad 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -92,7 +92,6 @@ public: void alter(const AlterCommands & params, ContextPtr alter_context, AlterLockHolder &) override; - Poco::Timestamp getUpdateTime() const; LoadablesConfigurationPtr getConfiguration() const; String getDictionaryName() const { return dictionary_name; } @@ -102,7 +101,6 @@ private: const Location location; mutable std::mutex dictionary_config_mutex; - Poco::Timestamp update_time; LoadablesConfigurationPtr configuration; scope_guard remove_repository_callback; From fd9231a886ffb37db3657695b590b43d0fbd31ef Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Mon, 18 Mar 2024 07:41:50 +0800 Subject: [PATCH 111/332] Add database = currentDatabase() condition to query test 03010_read_system_parts_table_test.sql The failed test suggest to add database = currentDatabase() condition to tests that queries on system.parts. --- .../0_stateless/03010_read_system_parts_table_test.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql index 6ec7e6212d5..8871822af4e 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -4,7 +4,7 @@ CREATE TABLE users (uid Int16, name String, age Int16) ENGINE=MergeTree ORDER BY INSERT INTO users VALUES (1231, 'John', 33); INSERT INTO users VALUES (6666, 'Ksenia', 48); -SELECT uuid, name from system.parts WHERE table = 'users'; +SELECT uuid, name from system.parts WHERE database = currentDatabase() AND table = 'users'; -SELECT table from system.parts WHERE uuid = '00000000-0000-0000-0000-000000000000'; +SELECT table from system.parts WHERE database = currentDatabase() AND uuid = '00000000-0000-0000-0000-000000000000'; DROP TABLE IF EXISTS users; From 1a61da1baefa5ae3f0f4d168a7357b4ce2221c82 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 02:16:52 +0100 Subject: [PATCH 112/332] Replace getFuzzerData with query text fuzzer in clickhouse-local --- CMakeLists.txt | 10 ++- programs/CMakeLists.txt | 4 - programs/local/CMakeLists.txt | 6 -- programs/local/LocalServer.cpp | 133 ++++++--------------------------- programs/main.cpp | 10 +-- src/Client/ClientBase.cpp | 25 +++++++ src/Client/ClientBase.h | 7 ++ src/Functions/CMakeLists.txt | 4 - 8 files changed, 66 insertions(+), 133 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4e16eace2..da29a9c93e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,8 @@ if (ENABLE_FUZZING) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) + + add_compile_definitions(FUZZING_MODE=1) endif() # Global libraries @@ -574,7 +576,7 @@ if (FUZZER) if (NOT(target_type STREQUAL "INTERFACE_LIBRARY" OR target_type STREQUAL "UTILITY")) target_compile_options(${target} PRIVATE "-fsanitize=fuzzer-no-link") endif() - if (target_type STREQUAL "EXECUTABLE" AND (target MATCHES ".+_fuzzer" OR target STREQUAL "clickhouse")) + if (target_type STREQUAL "EXECUTABLE" AND target MATCHES ".+_fuzzer") message(STATUS "${target} instrumented with fuzzer") target_link_libraries(${target} PUBLIC ch_contrib::fuzzer) # Add to fuzzers bundle @@ -583,6 +585,12 @@ if (FUZZER) get_target_property(target_bin_dir ${target} BINARY_DIR) add_custom_command(TARGET fuzzers POST_BUILD COMMAND mv "${target_bin_dir}/${target_bin_name}" "${CMAKE_CURRENT_BINARY_DIR}/programs/" VERBATIM) endif() + if (target STREQUAL "clickhouse") + message(STATUS "${target} instrumented with fuzzer") + target_link_libraries(${target} PUBLIC ch_contrib::fuzzer_no_main) + # Add to fuzzers bundle + add_dependencies(fuzzers ${target}) + endif() endif() endforeach() add_custom_command(TARGET fuzzers POST_BUILD COMMAND SRC=${CMAKE_SOURCE_DIR} BIN=${CMAKE_BINARY_DIR} OUT=${CMAKE_BINARY_DIR}/programs ${CMAKE_SOURCE_DIR}/tests/fuzz/build.sh VERBATIM) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index d945fdf4a6f..0d91de2dad8 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -268,10 +268,6 @@ if (ENABLE_TESTS) add_dependencies(clickhouse-bundle clickhouse-tests) endif() -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) -endif () - if (TARGET ch_contrib::protobuf) get_property(google_proto_files TARGET ch_contrib::protobuf PROPERTY google_proto_files) foreach (proto_file IN LISTS google_proto_files) diff --git a/programs/local/CMakeLists.txt b/programs/local/CMakeLists.txt index 565b67d0020..1aaa2859898 100644 --- a/programs/local/CMakeLists.txt +++ b/programs/local/CMakeLists.txt @@ -25,9 +25,3 @@ endif() # Always use internal readpassphrase target_link_libraries(clickhouse-local-lib PRIVATE readpassphrase) - -if (ENABLE_FUZZING) - add_compile_definitions(FUZZING_MODE=1) - set (WITH_COVERAGE ON) - target_link_libraries(clickhouse-local-lib PRIVATE ${LIB_FUZZING_ENGINE}) -endif () diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 167d361f5e3..4bb5d3a94b7 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -22,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -34,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -42,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -61,10 +56,6 @@ #include "config.h" -#if defined(FUZZING_MODE) - #include -#endif - #if USE_AZURE_BLOB_STORAGE # include #endif @@ -474,25 +465,10 @@ try } } -#if defined(FUZZING_MODE) - static bool first_time = true; - if (first_time) - { - - if (queries_files.empty() && queries.empty()) - { - std::cerr << "\033[31m" << "ClickHouse compiled in fuzzing mode." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "You have to provide a query with --query or --queries-file option." << "\033[0m" << std::endl; - std::cerr << "\033[31m" << "The query have to use function getFuzzerData() inside." << "\033[0m" << std::endl; - exit(1); - } - - is_interactive = false; -#else is_interactive = stdin_is_a_tty && (config().hasOption("interactive") || (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file"))); -#endif + if (!is_interactive) { /// We will terminate process on error @@ -538,15 +514,13 @@ try connect(); -#ifdef FUZZING_MODE - first_time = false; - } -#endif - String initial_query = getInitialCreateTableQuery(); if (!initial_query.empty()) processQueryText(initial_query); +#if defined(FUZZING_MODE) + runLibFuzzer(); +#else if (is_interactive && !delayed_interactive) { runInteractive(); @@ -558,10 +532,8 @@ try if (delayed_interactive) runInteractive(); } - -#ifndef FUZZING_MODE - cleanup(); #endif + return Application::EXIT_OK; } catch (const DB::Exception & e) @@ -829,23 +801,11 @@ void LocalServer::processConfig() void LocalServer::printHelpMessage([[maybe_unused]] const OptionsDescription & options_description) { -#if defined(FUZZING_MODE) - std::cout << - "usage: clickhouse -- \n" - "Note: It is important not to use only one letter keys with single dash for \n" - "for clickhouse-local arguments. It may work incorrectly.\n" - - "ClickHouse is build with coverage guided fuzzer (libfuzzer) inside it.\n" - "You have to provide a query which contains getFuzzerData function.\n" - "This will take the data from fuzzing engine, pass it to getFuzzerData function and execute a query.\n" - "Each time the data will be different, and it will last until some segfault or sanitizer assertion is found. \n"; -#else std::cout << getHelpHeader() << "\n"; std::cout << options_description.main_description.value() << "\n"; std::cout << getHelpFooter() << "\n"; std::cout << "In addition, --param_name=value can be specified for substitution of parameters for parametrized queries.\n"; std::cout << "\nSee also: https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/\n"; -#endif } @@ -921,6 +881,7 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; + /// Parameter arg after underline. if (arg.starts_with("--param_")) { @@ -952,9 +913,27 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum arg = argv[arg_num]; addMultiquery(arg, common_arguments); } + else if (arg == "--") + { +#if defined(FUZZING_MODE) + fuzzer_argc = 1 + argc - arg_num; + fuzzer_argv = argv + arg_num; + break; +#endif + } else + { common_arguments.emplace_back(arg); + } } + +#if defined(FUZZING_MODE) + if (!fuzzer_argc) + { + fuzzer_argc = 1; + fuzzer_argv = argv; + } +#endif } } @@ -988,67 +967,3 @@ int mainEntryClickHouseLocal(int argc, char ** argv) return code ? code : 1; } } - -#if defined(FUZZING_MODE) - -// linked from programs/main.cpp -bool isClickhouseApp(const std::string & app_suffix, std::vector & argv); - -std::optional fuzz_app; - -extern "C" int LLVMFuzzerInitialize(int * pargc, char *** pargv) -{ - std::vector argv(*pargv, *pargv + (*pargc + 1)); - - /// As a user you can add flags to clickhouse binary in fuzzing mode as follows - /// clickhouse local -- - - char **p = &(*pargv)[1]; - - auto it = argv.begin() + 1; - for (; *it; ++it) - { - if (strcmp(*it, "--") == 0) - { - ++it; - break; - } - } - - while (*it) - { - if (strncmp(*it, "--", 2) != 0) - { - *(p++) = *it; - it = argv.erase(it); - } - else - ++it; - } - - *pargc = static_cast(p - &(*pargv)[0]); - *p = nullptr; - - /// Initialize clickhouse-local app - fuzz_app.emplace(); - fuzz_app->init(static_cast(argv.size() - 1), argv.data()); - - return 0; -} - - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) -{ - try - { - auto input = String(reinterpret_cast(data), size); - DB::FunctionGetFuzzerData::update(input); - fuzz_app->run(); - } - catch (...) - { - } - - return 0; -} -#endif diff --git a/programs/main.cpp b/programs/main.cpp index 0a35594bd30..7162a18d764 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -67,7 +67,6 @@ namespace using MainFunc = int (*)(int, char**); -#if !defined(FUZZING_MODE) /// Add an item here to register new application std::pair clickhouse_applications[] = { @@ -111,7 +110,6 @@ int printHelp(int, char **) std::cerr << "clickhouse " << application.first << " [args] " << std::endl; return -1; } -#endif /// Add an item here to register a new short name std::pair clickhouse_short_names[] = @@ -284,7 +282,7 @@ struct Checker ; -#if !defined(FUZZING_MODE) && !defined(USE_MUSL) +#if !defined(USE_MUSL) /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete. void checkHarmfulEnvironmentVariables(char ** argv) { @@ -446,13 +444,8 @@ extern "C" /// /// extern bool inside_main; /// class C { C() { assert(inside_main); } }; -#ifndef FUZZING_MODE bool inside_main = false; -#else -bool inside_main = true; -#endif -#if !defined(FUZZING_MODE) int main(int argc_, char ** argv_) { inside_main = true; @@ -510,4 +503,3 @@ int main(int argc_, char ** argv_) return exit_code; } -#endif diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index d561a64895b..da0807d0c26 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2631,6 +2631,31 @@ void ClientBase::runNonInteractive() } +#if defined(FUZZING_MODE) +extern "C" int LLVMFuzzerRunDriver(int * argc, char *** argv, int (*callback)(const uint8_t * data, size_t size)); +ClientBase * app; + +void ClientBase::runLibFuzzer() +{ + app = this; + LLVMFuzzerRunDriver(&fuzzer_argc, &fuzzer_argv, [](const uint8_t * data, size_t size) + { + try + { + String query(reinterpret_cast(data), size); + app->processQueryText(query); + } + catch (...) + { + return -1; + } + + return 0; + }); +} +#endif + + void ClientBase::clearTerminal() { /// Clear from cursor until end of screen. diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 7a9e9666e67..48a779d1757 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -78,6 +78,13 @@ protected: void runInteractive(); void runNonInteractive(); +#if defined(FUZZING_MODE) + int fuzzer_argc = 0; + char ** fuzzer_argv = nullptr; + + void runLibFuzzer(); +#endif + virtual bool processWithFuzzing(const String &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Query processing with fuzzing is not implemented"); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index ac3e3671ae0..bfb1fc92df8 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -142,10 +142,6 @@ list (APPEND OBJECT_LIBS $ Date: Mon, 18 Mar 2024 02:16:24 +0100 Subject: [PATCH 113/332] Fix error in clickhouse-client --- src/Parsers/parseQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index 7aad0b010a5..51878efa706 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -92,7 +92,9 @@ void writeQueryWithHighlightedErrorPositions( } else { - size_t bytes_to_hilite = UTF8::seqLength(*current_position_to_hilite); + ssize_t bytes_to_hilite = UTF8::seqLength(*current_position_to_hilite); + if (bytes_to_hilite > end - current_position_to_hilite) + bytes_to_hilite = end - current_position_to_hilite; /// Bright on red background. out << "\033[41;1m"; From 2eb7de859d08a889db4d2e4f543a8c1699fcb5f4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 02:35:32 +0100 Subject: [PATCH 114/332] Correct arguments parsing --- programs/local/LocalServer.cpp | 16 ++++++---------- src/Client/ClientBase.cpp | 3 +++ src/Client/ClientBase.h | 4 +--- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 4bb5d3a94b7..f485e3880c1 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -878,6 +878,11 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &) { +#if defined(FUZZING_MODE) + if (argc) + fuzzer_args.push_back(argv[0]); +#endif + for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; @@ -916,8 +921,7 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum else if (arg == "--") { #if defined(FUZZING_MODE) - fuzzer_argc = 1 + argc - arg_num; - fuzzer_argv = argv + arg_num; + fuzzer_args.insert(fuzzer_args.end(), &argv[arg_num + 1], &argv[argc]); break; #endif } @@ -926,14 +930,6 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum common_arguments.emplace_back(arg); } } - -#if defined(FUZZING_MODE) - if (!fuzzer_argc) - { - fuzzer_argc = 1; - fuzzer_argv = argv; - } -#endif } } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index da0807d0c26..c4cb6e1a5ad 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2638,6 +2638,9 @@ ClientBase * app; void ClientBase::runLibFuzzer() { app = this; + int fuzzer_argc = fuzzer_args.size(); + char ** fuzzer_argv = fuzzer_args.data(); + LLVMFuzzerRunDriver(&fuzzer_argc, &fuzzer_argv, [](const uint8_t * data, size_t size) { try diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 48a779d1757..c0052ade7aa 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -79,9 +79,7 @@ protected: void runNonInteractive(); #if defined(FUZZING_MODE) - int fuzzer_argc = 0; - char ** fuzzer_argv = nullptr; - + std::vector fuzzer_args; void runLibFuzzer(); #endif From e27e6fb2f5410119b116bb12e9ff9c25d958d901 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 20 Feb 2024 11:41:08 +0000 Subject: [PATCH 115/332] Analyzer: support join using column from select list --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 140 ++++++++++++++---- src/Planner/PlannerJoinTree.cpp | 46 ++++-- src/Storages/StorageMerge.cpp | 8 +- tests/analyzer_tech_debt.txt | 1 - .../02989_join_using_parent_scope.reference | 88 +++++++++++ .../02989_join_using_parent_scope.sql | 45 ++++++ 6 files changed, 276 insertions(+), 52 deletions(-) create mode 100644 tests/queries/0_stateless/02989_join_using_parent_scope.reference create mode 100644 tests/queries/0_stateless/02989_join_using_parent_scope.sql diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1da5d86edf3..702a70ddc61 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1276,7 +1276,7 @@ private: return {}; } - static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional resolved_side) + static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional resolved_side, const ContextPtr & context) { if (resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN && JoinCommon::canBecomeNullable(resolved_identifier->getResultType()) && @@ -1285,7 +1285,14 @@ private: (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left))) { auto & resolved_column = resolved_identifier->as(); - resolved_column.setColumnType(makeNullableOrLowCardinalityNullable(resolved_column.getColumnType())); + auto new_result_type = makeNullableOrLowCardinalityNullable(resolved_column.getColumnType()); + resolved_column.setColumnType(new_result_type); + if (resolved_column.hasExpression()) + { + auto & resolved_expression = resolved_column.getExpression(); + if (!resolved_expression->getResultType()->equals(*new_result_type)) + resolved_expression = buildCastFunction(resolved_expression, new_result_type, context, true); + } } } @@ -3258,6 +3265,32 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id return {}; } +QueryTreeNodePtr checkIsMissedObjectJSONSubcolumn(const QueryTreeNodePtr & left_resolved_identifier, + const QueryTreeNodePtr & right_resolved_identifier) +{ + if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT + && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) + { + auto & left_resolved_column = left_resolved_identifier->as(); + auto & right_resolved_column = right_resolved_identifier->as(); + if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL") + return left_resolved_identifier; + } + else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) + { + auto & left_resolved_column = left_resolved_identifier->as(); + if (left_resolved_column.getValueStringRepresentation() == "NULL") + return left_resolved_identifier; + } + else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) + { + auto & right_resolved_column = right_resolved_identifier->as(); + if (right_resolved_column.getValueStringRepresentation() == "NULL") + return right_resolved_identifier; + } + return {}; +} + QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) @@ -3358,28 +3391,8 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo /// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced /// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here. - if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT - && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) - { - auto & left_resolved_column = left_resolved_identifier->as(); - auto & right_resolved_column = right_resolved_identifier->as(); - if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL") - return left_resolved_identifier; - } - else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) - { - resolved_side = JoinTableSide::Left; - auto & left_resolved_column = left_resolved_identifier->as(); - if (left_resolved_column.getValueStringRepresentation() == "NULL") - return left_resolved_identifier; - } - else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT) - { - resolved_side = JoinTableSide::Right; - auto & right_resolved_column = right_resolved_identifier->as(); - if (right_resolved_column.getValueStringRepresentation() == "NULL") - return right_resolved_identifier; - } + if (auto missed_subcolumn_identifier = checkIsMissedObjectJSONSubcolumn(left_resolved_identifier, right_resolved_identifier)) + return missed_subcolumn_identifier; if (left_resolved_identifier && right_resolved_identifier) { @@ -3522,7 +3535,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo if (scope.join_use_nulls) { resolved_identifier = resolved_identifier->clone(); - convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side); + convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope.context); } return resolved_identifier; @@ -4402,6 +4415,31 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as(); const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes(); + /** If column doesn't exists in the table, then do not match column from USING clause. + * Example: SELECT a + 1 AS id, * FROM (SELECT 1 AS a) AS t1 JOIN (SELECT 2 AS id) AS t2 USING (id); + * In this case `id` is not present in the left table expression, + * so asterisk should return `id` from the right table expression. + */ + auto is_column_from_parent_scope = [&scope](const QueryTreeNodePtr & using_node_from_table) + { + const auto & using_column_from_table = using_node_from_table->as(); + auto table_expression_data_it = scope.table_expression_node_to_data.find(using_column_from_table.getColumnSource()); + if (table_expression_data_it != scope.table_expression_node_to_data.end()) + { + const auto & table_expression_data = table_expression_data_it->second; + const auto & column_name = using_column_from_table.getColumnName(); + if (!table_expression_data.column_name_to_column_node.contains(column_name)) + { + return true; + } + } + return false; + }; + + if (is_column_from_parent_scope(join_using_column_nodes.at(0)) || + is_column_from_parent_scope(join_using_column_nodes.at(1))) + continue; + QueryTreeNodePtr matched_column_node; if (isRight(join_node->getKind())) @@ -4523,7 +4561,13 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I for (auto & [node, node_name] : matched_expression_nodes_with_names) { auto join_identifier_side = getColumnSideFromJoinTree(node, *nearest_scope_join_node); - convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side); + auto projection_name_it = node_to_projection_name.find(node); + node = node->clone(); + /// Set the same projection name for new nullable node + if (projection_name_it != node_to_projection_name.end()) + node_to_projection_name.emplace(node, projection_name_it->second); + + convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope.context); } } } @@ -7307,11 +7351,45 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); if (!result_left_table_expression) - throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, - "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", - join_node_typed.formatASTForErrorMessage(), - identifier_full_name, - scope.scope_node->formatASTForErrorMessage()); + { + /** Try to resolve identifier from parent subquery projection. + * Example: SELECT a + 1 AS v FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b + * In this case `b` is not in the left table expression, but it is in the parent subquery projection. + */ + QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; + if (query_node) + { + const auto & projection_list = query_node->getProjection(); + for (const auto & projection_node : projection_list.getNodes()) + { + if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) + { + auto left_subquery = std::make_shared(query_node->getMutableContext()); + left_subquery->getProjection().getNodes().push_back(projection_node->clone()); + left_subquery->getJoinTree() = join_node_typed.getLeftTableExpression(); + + IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/); + resolveQuery(left_subquery, left_subquery_scope); + + const auto & resolved_nodes = left_subquery->getProjection().getNodes(); + if (resolved_nodes.size() == 1) + { + /// Create ColumnNode with expression from parent projection + result_left_table_expression = std::make_shared( + NameAndTypePair{identifier_full_name, resolved_nodes.at(0)->getResultType()}, resolved_nodes.at(0), join_node_typed.getLeftTableExpression()); + break; + } + } + } + } + + if (!result_left_table_expression) + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); + } if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8ca8f0f258b..61a7a19f5be 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -585,6 +585,28 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info) return 0; } +std::unique_ptr createComputeAliasColumnsStep( + const std::unordered_map & alias_column_expressions, const DataStream & current_data_stream) +{ + ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared(current_data_stream.header.getColumnsWithTypeAndName()); + ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs(); + + for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions) + { + const auto & current_outputs = alias_column_actions_dag->getOutputs(); + action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end()); + merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag)); + } + + for (const auto * output_node : action_dag_outputs) + merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node); + merged_alias_columns_actions_dag->removeUnusedActions(false); + + auto alias_column_step = std::make_unique(current_data_stream, std::move(merged_alias_columns_actions_dag)); + alias_column_step->setStepDescription("Compute alias columns"); + return alias_column_step; +} + JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expression, const SelectQueryInfo & select_query_info, const SelectQueryOptions & select_query_options, @@ -865,22 +887,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions(); if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns) { - ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName()); - ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs(); - - for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions) - { - const auto & current_outputs = alias_column_actions_dag->getOutputs(); - action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end()); - merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag)); - } - - for (const auto * output_node : action_dag_outputs) - merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node); - merged_alias_columns_actions_dag->removeUnusedActions(false); - - auto alias_column_step = std::make_unique(query_plan.getCurrentDataStream(), std::move(merged_alias_columns_actions_dag)); - alias_column_step->setStepDescription("Compute alias columns"); + auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream()); query_plan.addStep(std::move(alias_column_step)); } @@ -961,6 +968,13 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres query_node_to_plan_step_mapping.insert(mapping.begin(), mapping.end()); query_plan = std::move(subquery_planner).extractQueryPlan(); } + + const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions(); + if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns) + { + auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream()); + query_plan.addStep(std::move(alias_column_step)); + } } else { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 9161b2773da..df3c141db1b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -678,13 +678,13 @@ public: void visitImpl(QueryTreeNodePtr & node) { - if (auto * column = node->as(); column != nullptr) + if (auto * column = node->as()) { if (column->hasExpression()) { - auto column_name = column->getColumnName(); - node = column->getExpressionOrThrow(); - node->setAlias(column_name); + QueryTreeNodePtr column_expression = column->getExpressionOrThrow(); + column_expression->setAlias(column->getColumnName()); + node = std::move(column_expression); } else column->setColumnSource(replacement_table_expression); diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index bcf0eee0d0e..21ab4a28ce4 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -1,7 +1,6 @@ 00223_shard_distributed_aggregation_memory_efficient 00717_merge_and_distributed 00725_memory_tracking -01062_pm_all_join_with_block_continuation 01083_expressions_in_engine_arguments 01155_rename_move_materialized_view 01584_distributed_buffer_cannot_find_column diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference new file mode 100644 index 00000000000..c396431049f --- /dev/null +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference @@ -0,0 +1,88 @@ +-- { echoOn } +SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL; +1 +1 +1 +1 +SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL; +2 +3 +4 +SELECT b + 2 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL; +2 +3 +SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL; +2 +3 +4 +5 +SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL; +0 0 +1 1 +2 2 +3 3 +SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER } +-- SELECT * works returns all columns from both tables in new analyzer +SET allow_experimental_analyzer = 1; +SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +3 3 0 3 +3 3 1 3 +3 3 2 3 +3 3 3 3 +3 3 \N 0 +3 3 \N 1 +3 3 \N 2 +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +1 0 1 +2 1 2 +3 2 3 +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +1 0 1 +2 1 2 +3 2 3 +SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL; +1 0 1 +2 1 2 +3 2 3 +4 3 \N +SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL; +1 0 1 +2 1 2 +3 2 3 +\N \N 0 +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +1 0 1 +2 1 2 +3 2 3 +4 3 \N +\N \N 0 +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1; +1 0 1 2 3 +2 1 2 3 4 +3 2 3 4 5 +4 3 \N \N \N +\N \N 0 1 2 +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +1 0 1 2 +2 1 2 3 +3 2 3 4 +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +1 0 1 2 +2 1 2 3 +3 2 3 4 +4 3 \N \N +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +1 0 1 2 +2 1 2 3 +3 2 3 4 +\N \N 0 1 +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +1 0 1 2 +2 1 2 3 +3 2 3 4 +4 3 \N \N +\N \N 0 1 +DROP TABLE IF EXISTS tabc; +DROP TABLE IF EXISTS ta; +DROP TABLE IF EXISTS tb; +DROP TABLE IF EXISTS tc; diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql new file mode 100644 index 00000000000..05f1137f60e --- /dev/null +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql @@ -0,0 +1,45 @@ +DROP TABLE IF EXISTS tabc; +CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1) ENGINE = MergeTree ORDER BY a; +INSERT INTO tabc SELECT number FROM numbers(4); + +DROP TABLE IF EXISTS ta; +CREATE TABLE ta (a Int32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO ta SELECT number FROM numbers(4); + +DROP TABLE IF EXISTS tb; +CREATE TABLE tb (b Int32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tb SELECT number FROM numbers(4); + +SET join_use_nulls = 1; + +-- { echoOn } +SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL; +SELECT b + 2 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL; + +SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER } + +-- SELECT * works returns all columns from both tables in new analyzer +SET allow_experimental_analyzer = 1; + +SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; + +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1; + +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; + + +DROP TABLE IF EXISTS tabc; +DROP TABLE IF EXISTS ta; +DROP TABLE IF EXISTS tb; +DROP TABLE IF EXISTS tc; From fbad9b799cd4d8549bf8479c94688cb7a1277c89 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 11 Mar 2024 17:45:05 +0000 Subject: [PATCH 116/332] fix PREWHERE with join_use_nulls --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 90 +++++++++++++------ .../02989_join_using_parent_scope.reference | 68 +++++++------- .../02989_join_using_parent_scope.sql | 5 +- 3 files changed, 102 insertions(+), 61 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 702a70ddc61..35b6f05191e 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -776,6 +776,7 @@ struct IdentifierResolveScope std::unordered_map table_expression_node_to_data; QueryTreeNodePtrWithHashSet nullable_group_by_keys; + QueryTreeNodePtrWithHashMap nullable_join_columns; /// Use identifier lookup to result cache bool use_identifier_lookup_to_result_cache = true; @@ -1276,7 +1277,11 @@ private: return {}; } - static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional resolved_side, const ContextPtr & context) + static QueryTreeNodePtr convertJoinedColumnTypeToNullIfNeeded( + const QueryTreeNodePtr & resolved_identifier, + const JoinKind & join_kind, + std::optional resolved_side, + IdentifierResolveScope & scope) { if (resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN && JoinCommon::canBecomeNullable(resolved_identifier->getResultType()) && @@ -1284,16 +1289,20 @@ private: (isLeft(join_kind) && resolved_side && *resolved_side == JoinTableSide::Right) || (isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left))) { - auto & resolved_column = resolved_identifier->as(); + auto nullable_resolved_identifier = resolved_identifier->clone(); + auto & resolved_column = nullable_resolved_identifier->as(); auto new_result_type = makeNullableOrLowCardinalityNullable(resolved_column.getColumnType()); resolved_column.setColumnType(new_result_type); if (resolved_column.hasExpression()) { auto & resolved_expression = resolved_column.getExpression(); if (!resolved_expression->getResultType()->equals(*new_result_type)) - resolved_expression = buildCastFunction(resolved_expression, new_result_type, context, true); + resolved_expression = buildCastFunction(resolved_expression, new_result_type, scope.context, true); } + scope.nullable_join_columns[nullable_resolved_identifier] = resolved_identifier; + return nullable_resolved_identifier; } + return nullptr; } /// Resolve identifier functions @@ -3534,8 +3543,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo if (scope.join_use_nulls) { - resolved_identifier = resolved_identifier->clone(); - convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope.context); + auto nullable_resolved_identifier = convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope); + if (nullable_resolved_identifier) + resolved_identifier = nullable_resolved_identifier; } return resolved_identifier; @@ -4428,10 +4438,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( { const auto & table_expression_data = table_expression_data_it->second; const auto & column_name = using_column_from_table.getColumnName(); - if (!table_expression_data.column_name_to_column_node.contains(column_name)) - { - return true; - } + return !table_expression_data.column_name_to_column_node.contains(column_name); } return false; }; @@ -4562,12 +4569,14 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I { auto join_identifier_side = getColumnSideFromJoinTree(node, *nearest_scope_join_node); auto projection_name_it = node_to_projection_name.find(node); - node = node->clone(); - /// Set the same projection name for new nullable node - if (projection_name_it != node_to_projection_name.end()) - node_to_projection_name.emplace(node, projection_name_it->second); - - convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope.context); + auto nullable_node = convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope); + if (nullable_node) + { + node = nullable_node; + /// Set the same projection name for new nullable node + if (projection_name_it != node_to_projection_name.end()) + node_to_projection_name.emplace(node, projection_name_it->second); + } } } } @@ -7526,6 +7535,29 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, scope.table_expressions_in_resolve_process.erase(join_tree_node.get()); } +class ReplaceColumnsVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit ReplaceColumnsVisitor(const QueryTreeNodePtrWithHashMap & replacement_map_, const ContextPtr & context_) + : replacement_map(replacement_map_) + , context(context_) + {} + + void visitImpl(QueryTreeNodePtr & node) + { + if (auto it = replacement_map.find(node); it != replacement_map.end()) + node = it->second; + if (auto * function_node = node->as()) + rerunFunctionResolve(function_node, context); + } + + bool shouldTraverseTopToBottom() const { return false; } + +private: + const QueryTreeNodePtrWithHashMap & replacement_map; + const ContextPtr & context; +}; + /** Resolve query. * This function modifies query node during resolve. It is caller responsibility to clone query node before resolve * if it is needed for later use. @@ -7713,21 +7745,23 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier scope.scope_node->formatASTForErrorMessage()); } - if (query_node_typed.getPrewhere()) + if (auto & prewhere_node = query_node_typed.getPrewhere()) { - /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls. - * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1 - * Column `a` should be resolved from table and should not change its type to Nullable. - */ - bool join_use_nulls = scope.join_use_nulls; - bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache; - scope.join_use_nulls = false; - scope.use_identifier_lookup_to_result_cache = false; + resolveExpressionNode(prewhere_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - - scope.join_use_nulls = join_use_nulls; - scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache; + if (scope.join_use_nulls) + { + /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls. + * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE b = 1 + * Column `a` should be resolved from table and should not change its type to Nullable. + * More complicated example when column is somewhere inside an expression: + * SELECT a + 1 as b FROM t1 JOIN t2 USING (id) PREWHERE b = 1 + * expression `a + 1 as b` in projection and in PREWHERE should have different `a`. + */ + prewhere_node = prewhere_node->clone(); + ReplaceColumnsVisitor replace_visitor(scope.nullable_join_columns, scope.context); + replace_visitor.visit(prewhere_node); + } } if (query_node_typed.getWhere()) diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference index c396431049f..cf1ae93936f 100644 --- a/tests/queries/0_stateless/02989_join_using_parent_scope.reference +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference @@ -25,43 +25,43 @@ SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER } -- SELECT * works returns all columns from both tables in new analyzer SET allow_experimental_analyzer = 1; SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; -3 3 0 3 -3 3 1 3 -3 3 2 3 -3 3 3 3 -3 3 \N 0 -3 3 \N 1 -3 3 \N 2 +3 3 0 3 abc3 +3 3 1 3 abc3 +3 3 2 3 abc3 +3 3 3 3 abc3 +3 3 \N 0 abc0 +3 3 \N 1 abc1 +3 3 \N 2 abc2 SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; -1 0 1 -2 1 2 -3 2 3 +1 0 1 abc1 +2 1 2 abc2 +3 2 3 abc3 SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; -1 0 1 -2 1 2 -3 2 3 +1 0 1 abc1 +2 1 2 abc2 +3 2 3 abc3 SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL; -1 0 1 -2 1 2 -3 2 3 -4 3 \N +1 0 1 abc1 +2 1 2 abc2 +3 2 3 abc3 +4 3 \N \N SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL; -1 0 1 -2 1 2 -3 2 3 -\N \N 0 +1 0 1 abc1 +2 1 2 abc2 +3 2 3 abc3 +\N \N 0 abc0 SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; -1 0 1 -2 1 2 -3 2 3 -4 3 \N -\N \N 0 +1 0 1 abc1 +2 1 2 abc2 +3 2 3 abc3 +4 3 \N \N +\N \N 0 abc0 SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1; -1 0 1 2 3 -2 1 2 3 4 -3 2 3 4 5 -4 3 \N \N \N -\N \N 0 1 2 +1 0 1 abc1 2 3 +2 1 2 abc2 3 4 +3 2 3 abc3 4 5 +4 3 \N \N \N \N +\N \N 0 abc0 1 2 SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; 1 0 1 2 2 1 2 3 @@ -82,6 +82,12 @@ SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc 3 2 3 4 4 3 \N \N \N \N 0 1 +SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL; +3 abc3 +4 \N +\N abc0 +\N abc1 +\N abc2 DROP TABLE IF EXISTS tabc; DROP TABLE IF EXISTS ta; DROP TABLE IF EXISTS tb; diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql index 05f1137f60e..9c8501f6b00 100644 --- a/tests/queries/0_stateless/02989_join_using_parent_scope.sql +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS tabc; -CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1) ENGINE = MergeTree ORDER BY a; -INSERT INTO tabc SELECT number FROM numbers(4); +CREATE TABLE tabc (a UInt32, b UInt32 ALIAS a + 1, c UInt32 ALIAS b + 1, s String) ENGINE = MergeTree ORDER BY a; +INSERT INTO tabc (a, s) SELECT number, 'abc' || toString(number) FROM numbers(4); DROP TABLE IF EXISTS ta; CREATE TABLE ta (a Int32) ENGINE = MergeTree ORDER BY tuple(); @@ -38,6 +38,7 @@ SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL; DROP TABLE IF EXISTS tabc; DROP TABLE IF EXISTS ta; From dbed5d1902eee94d2db9596c74ccf0b7335bb293 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 13 Mar 2024 12:14:14 +0000 Subject: [PATCH 117/332] Add analyzer_compatibility_join_using_top_level_identifier --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 90 ++++++++++++------- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + .../02989_join_using_parent_scope.reference | 70 ++++++++++++--- .../02989_join_using_parent_scope.sql | 68 +++++++++++--- 5 files changed, 168 insertions(+), 62 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 35b6f05191e..0414ea93f94 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7357,48 +7357,70 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS join_using_identifiers.insert(identifier_full_name); - IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; - auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); - if (!result_left_table_expression) + const auto & settings = scope.context->getSettingsRef(); + + /** While resolving JOIN USING identifier, try to resolve identifier from parent subquery projection. + * Example: SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b + * In this case `b` is not in the left table expression, but it is in the parent subquery projection. + */ + auto try_resolve_identifier_from_query_projection = [this](const String & identifier_full_name_, + const QueryTreeNodePtr & left_table_expression, + const IdentifierResolveScope & scope_) -> QueryTreeNodePtr { - /** Try to resolve identifier from parent subquery projection. - * Example: SELECT a + 1 AS v FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b - * In this case `b` is not in the left table expression, but it is in the parent subquery projection. - */ - QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; - if (query_node) + const QueryNode * query_node = scope_.scope_node ? scope_.scope_node->as() : nullptr; + if (!query_node) + return nullptr; + + const auto & projection_list = query_node->getProjection(); + for (const auto & projection_node : projection_list.getNodes()) { - const auto & projection_list = query_node->getProjection(); - for (const auto & projection_node : projection_list.getNodes()) + if (projection_node->hasAlias() && identifier_full_name_ == projection_node->getAlias()) { - if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) + auto left_subquery = std::make_shared(query_node->getMutableContext()); + left_subquery->getProjection().getNodes().push_back(projection_node->clone()); + left_subquery->getJoinTree() = left_table_expression; + + IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/); + resolveQuery(left_subquery, left_subquery_scope); + + const auto & resolved_nodes = left_subquery->getProjection().getNodes(); + if (resolved_nodes.size() == 1) { - auto left_subquery = std::make_shared(query_node->getMutableContext()); - left_subquery->getProjection().getNodes().push_back(projection_node->clone()); - left_subquery->getJoinTree() = join_node_typed.getLeftTableExpression(); - - IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/); - resolveQuery(left_subquery, left_subquery_scope); - - const auto & resolved_nodes = left_subquery->getProjection().getNodes(); - if (resolved_nodes.size() == 1) - { - /// Create ColumnNode with expression from parent projection - result_left_table_expression = std::make_shared( - NameAndTypePair{identifier_full_name, resolved_nodes.at(0)->getResultType()}, resolved_nodes.at(0), join_node_typed.getLeftTableExpression()); - break; - } + /// Create ColumnNode with expression from parent projection + return std::make_shared( + NameAndTypePair{identifier_full_name_, resolved_nodes.front()->getResultType()}, + resolved_nodes.front(), left_table_expression); } } } + return nullptr; + }; - if (!result_left_table_expression) - throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, - "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", - join_node_typed.formatASTForErrorMessage(), - identifier_full_name, - scope.scope_node->formatASTForErrorMessage()); - } + QueryTreeNodePtr result_left_table_expression = nullptr; + /** With `analyzer_compatibility_join_using_top_level_identifier` alias in projection has higher priority than column from left table. + * But if aliased expression cannot be resolved from left table, we get UNKNOW_IDENTIFIER error, + * despite the fact that column from USING could be resolved from left table. + * It's compatibility with a default behavior for old analyzer. + */ + if (settings.analyzer_compatibility_join_using_top_level_identifier) + result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); + + IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; + if (!result_left_table_expression) + result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); + + /// Here we may try to resolve identifier from projection in case it's not resolved from left table expression + /// and analyzer_compatibility_join_using_top_level_identifier is disabled. + /// For now we do not do this, because not all corner cases are clear. + /// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression) + /// result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); + + if (!result_left_table_expression) + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, + "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", + join_node_typed.formatASTForErrorMessage(), + identifier_full_name, + scope.scope_node->formatASTForErrorMessage()); if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e6adb00137f..bad130d6ef4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -371,6 +371,7 @@ class IColumn; M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ + M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \ \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 8e2b2915c2a..07a1c88363a 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -96,6 +96,7 @@ static std::map sett {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, + {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference index cf1ae93936f..965bfaf7c43 100644 --- a/tests/queries/0_stateless/02989_join_using_parent_scope.reference +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference @@ -22,9 +22,8 @@ SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL; 2 2 3 3 SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER } --- SELECT * works returns all columns from both tables in new analyzer -SET allow_experimental_analyzer = 1; -SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +-- SELECT * returns all columns from both tables in new analyzer +SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 3 3 0 3 abc3 3 3 1 3 abc3 3 3 2 3 abc3 @@ -32,62 +31,105 @@ SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; 3 3 \N 0 abc0 3 3 \N 1 abc1 3 3 \N 2 abc2 -SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 abc1 2 1 2 abc2 3 2 3 abc3 -SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 abc1 2 1 2 abc2 3 2 3 abc3 -SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 abc1 2 1 2 abc2 3 2 3 abc3 4 3 \N \N -SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 abc1 2 1 2 abc2 3 2 3 abc3 \N \N 0 abc0 -SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 abc1 2 1 2 abc2 3 2 3 abc3 4 3 \N \N \N \N 0 abc0 -SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1; 1 0 1 abc1 2 3 2 1 2 abc2 3 4 3 2 3 abc3 4 5 4 3 \N \N \N \N \N \N 0 abc0 1 2 -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 2 2 1 2 3 3 2 3 4 -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 2 2 1 2 3 3 2 3 4 4 3 \N \N -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 2 2 1 2 3 3 2 3 4 \N \N 0 1 -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 1 0 1 2 2 1 2 3 3 2 3 4 4 3 \N \N \N \N 0 1 -SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL; +SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; 3 abc3 4 \N \N abc0 \N abc1 \N abc2 +-- It's a default behavior for old analyzer and new with analyzer_compatibility_join_using_top_level_identifier +-- Column `b` actually exists in left table, but `b` from USING is resoled to `a + 2` and `a` is not in left table +-- so we get UNKNOWN_IDENTIFIER error. +SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { serverError UNKNOWN_IDENTIFIER } +-- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table +SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1; +2 +3 +4 +-- This is example where query may return different results with different `analyzer_compatibility_join_using_top_level_identifier` + +DROP TABLE IF EXISTS users; +CREATE TABLE users (uid Int16, name String, spouse_name String) ENGINE=Memory; +INSERT INTO users VALUES (1231, 'John', 'Ksenia'); +INSERT INTO users VALUES (6666, 'Ksenia', ''); +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1; +uid name u2.uid u2.name +Int16 String Int16 String +1231 Ksenia 6666 Ksenia +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0; +uid name u2.uid u2.name +Int16 String Int16 String +1231 Ksenia 1231 John +6666 6666 Ksenia +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 0; +uid name u2.uid u2.name +Int16 String Int16 String +1231 Ksenia 6666 Ksenia +DROP TABLE IF EXISTS users; DROP TABLE IF EXISTS tabc; DROP TABLE IF EXISTS ta; DROP TABLE IF EXISTS tb; diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql index 9c8501f6b00..2e4d9f097f7 100644 --- a/tests/queries/0_stateless/02989_join_using_parent_scope.sql +++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql @@ -12,6 +12,8 @@ INSERT INTO tb SELECT number FROM numbers(4); SET join_use_nulls = 1; +SET analyzer_compatibility_join_using_top_level_identifier = 1; + -- { echoOn } SELECT 1 AS a FROM tb JOIN tabc USING (a) ORDER BY ALL; SELECT a + 2 AS b FROM ta JOIN tabc USING (b) ORDER BY ALL; @@ -21,24 +23,62 @@ SELECT a + 2 AS c FROM ta JOIN tabc USING (c) ORDER BY ALL; SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL; SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER } --- SELECT * works returns all columns from both tables in new analyzer -SET allow_experimental_analyzer = 1; +-- SELECT * returns all columns from both tables in new analyzer +SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; -SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; +SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1; -SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; +SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; -SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL; +SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1; + + +-- It's a default behavior for old analyzer and new with analyzer_compatibility_join_using_top_level_identifier +-- Column `b` actually exists in left table, but `b` from USING is resoled to `a + 2` and `a` is not in left table +-- so we get UNKNOWN_IDENTIFIER error. +SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { serverError UNKNOWN_IDENTIFIER } + +-- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table +SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1; + +-- This is example where query may return different results with different `analyzer_compatibility_join_using_top_level_identifier` + +DROP TABLE IF EXISTS users; +CREATE TABLE users (uid Int16, name String, spouse_name String) ENGINE=Memory; + +INSERT INTO users VALUES (1231, 'John', 'Ksenia'); +INSERT INTO users VALUES (6666, 'Ksenia', ''); + +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1; + +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0; + +SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name +FROM users u1 JOIN users u2 USING (name) +ORDER BY u1.uid +FORMAT TSVWithNamesAndTypes +SETTINGS allow_experimental_analyzer = 0; + +DROP TABLE IF EXISTS users; -SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL; DROP TABLE IF EXISTS tabc; DROP TABLE IF EXISTS ta; From 589c784e7bddfba89248bde6a2957457f9fa2538 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 18 Mar 2024 11:12:14 +0000 Subject: [PATCH 118/332] upd 01062_pm_all_join_with_block_continuation --- .../01062_pm_all_join_with_block_continuation.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql b/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql index 15a28e92386..6a42c725e4a 100644 --- a/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql +++ b/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql @@ -1,8 +1,8 @@ SET max_memory_usage = 50000000; SET join_algorithm = 'partial_merge'; - +SET analyzer_compatibility_join_using_top_level_identifier = 1; SELECT 'defaults'; - + SELECT count(1) FROM ( SELECT materialize(1) as k, n FROM numbers(10) nums JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j @@ -20,7 +20,7 @@ SELECT count(1), uniqExact(n) FROM ( -- errors SET max_joined_block_size_rows = 0; - + SELECT count(1) FROM ( SELECT materialize(1) as k, n FROM numbers(10) nums JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j From 9065f34534a78ebc09cd0091ad18e8e5cac7ba46 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:26:05 +0100 Subject: [PATCH 119/332] Fix infinite loop in function `hop` --- src/Functions/FunctionsTimeWindow.cpp | 205 +++++++++++------- src/Functions/FunctionsTimeWindow.h | 35 --- .../03013_hop_infinite_loop.reference | 0 .../0_stateless/03013_hop_infinite_loop.sql | 2 + 4 files changed, 126 insertions(+), 116 deletions(-) create mode 100644 tests/queries/0_stateless/03013_hop_infinite_loop.reference create mode 100644 tests/queries/0_stateless/03013_hop_infinite_loop.sql diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 017661231f5..91eb70fdd66 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -27,87 +27,124 @@ namespace ErrorCodes namespace { - std::tuple - dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const String & function_name) - { - const auto * interval_type = checkAndGetDataType(interval_column.type.get()); - if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - interval_column.name, function_name); - const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); - if (!interval_column_const_int64) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - interval_column.name, function_name); - Int64 num_units = interval_column_const_int64->getValue(); - if (num_units <= 0) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for column {} of function {} must be positive", - interval_column.name, function_name); - return {interval_type->getKind(), num_units}; - } +std::tuple +dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const String & function_name) +{ + const auto * interval_type = checkAndGetDataType(interval_column.type.get()); + if (!interval_type) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", + interval_column.name, function_name); + const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); + if (!interval_column_const_int64) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", + interval_column.name, function_name); + Int64 num_units = interval_column_const_int64->getValue(); + if (num_units <= 0) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Value for column {} of function {} must be positive", + interval_column.name, function_name); - ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) + return {interval_type->getKind(), num_units}; +} + +ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) +{ + if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) { - if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) - { - if (!checkColumn>(*col_tuple->getColumnPtr(index))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " - "Must be a Tuple(DataTime, DataTime)", function_name); - return col_tuple->getColumnPtr(index); - } - else - { + if (!checkColumn>(*col_tuple->getColumnPtr(index))) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " - "Must be Tuple", function_name); - } + "Must be a Tuple(DataTime, DataTime)", function_name); + return col_tuple->getColumnPtr(index); } - - void checkFirstArgument(const ColumnWithTypeAndName & argument, const String & function_name) + else { - if (!isDateTime(argument.type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be a date with time", argument.type->getName(), function_name); - } - - void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) - { - const auto * interval_type = checkAndGetDataType(argument.type.get()); - if (!interval_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "Should be an interval of time", argument.type->getName(), function_name); - interval_kind = interval_type->getKind(); - result_type_is_date = (interval_type->getKind() == IntervalKind::Kind::Year) || (interval_type->getKind() == IntervalKind::Kind::Quarter) - || (interval_type->getKind() == IntervalKind::Kind::Month) || (interval_type->getKind() == IntervalKind::Kind::Week); - } - - void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, bool & result_type_is_date) - { - IntervalKind interval_kind; - checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); - } - - void checkTimeZoneArgument( - const ColumnWithTypeAndName & argument, - const String & function_name) - { - if (!WhichDataType(argument.type).isString()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " - "This argument is optional and must be a constant string with timezone name", - argument.type->getName(), function_name); - } - - bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) - { - if (WhichDataType(argument.type).isString()) - { - checkTimeZoneArgument(argument, function_name); - return false; - } - checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); - return true; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " + "Must be Tuple", function_name); } } +void checkFirstArgument(const ColumnWithTypeAndName & argument, const String & function_name) +{ + if (!isDateTime(argument.type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "Should be a date with time", argument.type->getName(), function_name); +} + +void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) +{ + const auto * interval_type = checkAndGetDataType(argument.type.get()); + if (!interval_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "Should be an interval of time", argument.type->getName(), function_name); + interval_kind = interval_type->getKind(); + result_type_is_date = (interval_type->getKind() == IntervalKind::Kind::Year) || (interval_type->getKind() == IntervalKind::Kind::Quarter) + || (interval_type->getKind() == IntervalKind::Kind::Month) || (interval_type->getKind() == IntervalKind::Kind::Week); +} + +void checkIntervalArgument(const ColumnWithTypeAndName & argument, const String & function_name, bool & result_type_is_date) +{ + IntervalKind interval_kind; + checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); +} + +void checkTimeZoneArgument( + const ColumnWithTypeAndName & argument, + const String & function_name) +{ + if (!WhichDataType(argument.type).isString()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}. " + "This argument is optional and must be a constant string with timezone name", + argument.type->getName(), function_name); +} + +bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, const String & function_name, IntervalKind & interval_kind, bool & result_type_is_date) +{ + if (WhichDataType(argument.type).isString()) + { + checkTimeZoneArgument(argument, function_name); + return false; + } + checkIntervalArgument(argument, function_name, interval_kind, result_type_is_date); + return true; +} + +template +struct TimeWindowImpl +{ + static constexpr auto name = "UNKNOWN"; + + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); + + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); +}; + +template +class FunctionTimeWindow : public IFunction +{ +public: + static constexpr auto name = TimeWindowImpl::name; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; +}; + +using FunctionTumble = FunctionTimeWindow; +using FunctionTumbleStart = FunctionTimeWindow; +using FunctionTumbleEnd = FunctionTimeWindow; +using FunctionHop = FunctionTimeWindow; +using FunctionWindowId = FunctionTimeWindow; +using FunctionHopStart = FunctionTimeWindow; +using FunctionHopEnd = FunctionTimeWindow; + + template <> struct TimeWindowImpl { @@ -393,6 +430,9 @@ struct TimeWindowImpl wstart = AddTime::execute(wend, -window_num_units, time_zone); ToType wend_latest; + if (wstart > wend) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time overflow in function {}", name); + do { wend_latest = wend; @@ -414,7 +454,7 @@ struct TimeWindowImpl { static constexpr auto name = "windowID"; - [[maybe_unused]] static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) { bool result_type_is_date; IntervalKind interval_kind_1; @@ -456,8 +496,7 @@ struct TimeWindowImpl return std::make_shared(); } - [[maybe_unused]] static ColumnPtr - dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto & hop_interval_column = arguments[1]; @@ -527,6 +566,9 @@ struct TimeWindowImpl ToType wend = AddTime::execute(wstart, hop_num_units, time_zone); ToType wend_latest; + if (wstart > wend) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time overflow in function {}", name); + do { wend_latest = wend; @@ -538,14 +580,13 @@ struct TimeWindowImpl return end; } - [[maybe_unused]] static ColumnPtr - dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { ColumnPtr column = TimeWindowImpl::dispatchForColumns(arguments, function_name); return executeWindowBound(column, 1, function_name); } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { if (arguments.size() == 2) return dispatchForTumbleColumns(arguments, function_name); @@ -585,7 +626,7 @@ struct TimeWindowImpl } } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto which_type = WhichDataType(time_column.type); @@ -608,12 +649,12 @@ struct TimeWindowImpl { static constexpr auto name = "hopEnd"; - [[maybe_unused]] static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) + static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name) { return TimeWindowImpl::getReturnType(arguments, function_name); } - [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) + static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name) { const auto & time_column = arguments[0]; const auto which_type = WhichDataType(time_column.type); @@ -644,6 +685,8 @@ ColumnPtr FunctionTimeWindow::executeImpl(const ColumnsWithTypeAndName & a return TimeWindowImpl::dispatchForColumns(arguments, name); } +} + REGISTER_FUNCTION(TimeWindow) { factory.registerFunction(); diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 658963cb69c..ab8deef08c0 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -156,39 +156,4 @@ template <> \ ADD_SUBSECONDS(Nanosecond, 9) #undef ADD_SUBSECONDS -template -struct TimeWindowImpl -{ - static constexpr auto name = "UNKNOWN"; - - static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); - - static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); -}; - -template -class FunctionTimeWindow : public IFunction -{ -public: - static constexpr auto name = TimeWindowImpl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; -}; - -using FunctionTumble = FunctionTimeWindow; -using FunctionTumbleStart = FunctionTimeWindow; -using FunctionTumbleEnd = FunctionTimeWindow; -using FunctionHop = FunctionTimeWindow; -using FunctionWindowId = FunctionTimeWindow; -using FunctionHopStart = FunctionTimeWindow; -using FunctionHopEnd = FunctionTimeWindow; } diff --git a/tests/queries/0_stateless/03013_hop_infinite_loop.reference b/tests/queries/0_stateless/03013_hop_infinite_loop.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03013_hop_infinite_loop.sql b/tests/queries/0_stateless/03013_hop_infinite_loop.sql new file mode 100644 index 00000000000..bdac87f5af1 --- /dev/null +++ b/tests/queries/0_stateless/03013_hop_infinite_loop.sql @@ -0,0 +1,2 @@ +SET allow_experimental_window_view = 1; +SELECT hop(toDateTime(0), INTERVAL 1 DAY, INTERVAL 3 DAY); -- { serverError BAD_ARGUMENTS } From 8434f049755ff8ea438d6d8c3cef1ad8f70cb2d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:40:49 +0100 Subject: [PATCH 120/332] Fix style --- src/Functions/FunctionsTimeWindow.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 91eb70fdd66..97608db51e8 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; extern const int SYNTAX_ERROR; + extern const int BAD_ARGUMENTS; } namespace From 97aca12830f98890a6e2d71b2c7e9ce149d72cdb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:58:03 +0100 Subject: [PATCH 121/332] Fix crash in window view --- src/Functions/FunctionsTimeWindow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 97608db51e8..be51978499b 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -48,11 +48,12 @@ dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column, const return {interval_type->getKind(), num_units}; } -ColumnPtr executeWindowBound(const ColumnPtr & column, int index, const String & function_name) +ColumnPtr executeWindowBound(const ColumnPtr & column, size_t index, const String & function_name) { + chassert(index == 0 || index == 1); if (const ColumnTuple * col_tuple = checkAndGetColumn(column.get()); col_tuple) { - if (!checkColumn>(*col_tuple->getColumnPtr(index))) + if (index >= col_tuple->tupleSize() || !checkColumn>(*col_tuple->getColumnPtr(index))) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for first argument of function {}. " "Must be a Tuple(DataTime, DataTime)", function_name); return col_tuple->getColumnPtr(index); From 9fa449fc75d1a9b69aa59737b5dd4fe0bc25dace Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 12:59:09 +0100 Subject: [PATCH 122/332] Add a test --- tests/queries/0_stateless/03014_window_view_crash.reference | 0 tests/queries/0_stateless/03014_window_view_crash.sql | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/03014_window_view_crash.reference create mode 100644 tests/queries/0_stateless/03014_window_view_crash.sql diff --git a/tests/queries/0_stateless/03014_window_view_crash.reference b/tests/queries/0_stateless/03014_window_view_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03014_window_view_crash.sql b/tests/queries/0_stateless/03014_window_view_crash.sql new file mode 100644 index 00000000000..ff9835521f5 --- /dev/null +++ b/tests/queries/0_stateless/03014_window_view_crash.sql @@ -0,0 +1,2 @@ +SET allow_experimental_window_view = 1; +SELECT hopEnd(tuple([[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]], [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]], [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]])::Tuple(MultiPolygon)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } From 64b5acf0e47b65f616dce2427a57014e1e80d534 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 14 Mar 2024 18:38:27 +0100 Subject: [PATCH 123/332] Code cleanup --- .../ClusterProxy/SelectStreamFactory.h | 3 -- .../OptimizeShardingKeyRewriteInVisitor.cpp | 28 +++++++++---------- .../OptimizeShardingKeyRewriteInVisitor.h | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 760281284fd..2e7b2445c6b 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -44,8 +43,6 @@ ASTPtr rewriteSelectQuery( using ColumnsDescriptionByShardNum = std::unordered_map; using AdditionalShardFilterGenerator = std::function; -using MissingObjectList = std::vector>; - class SelectStreamFactory { public: diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 0df6fcee229..84279282d92 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -1,19 +1,17 @@ -#include -#include -#include -#include -#include -#include -#include #include -#include "Analyzer/ColumnNode.h" -#include "Analyzer/ConstantNode.h" -#include "Analyzer/FunctionNode.h" -#include "Analyzer/IQueryTreeNode.h" -#include "Analyzer/InDepthQueryTreeVisitor.h" -#include "Analyzer/Utils.h" -#include "DataTypes/IDataType.h" -#include "Interpreters/Context_fwd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace { diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h index d202609160b..1b2d5a8fc15 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include "Analyzer/IQueryTreeNode.h" namespace DB { From 08d64f9e118329232a2149ff21e6ad47ffd37e1c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 00:42:12 +0100 Subject: [PATCH 124/332] Get rid of modifying a shared AbstractConfiguration (between StorageDictionary and ExternalDictionariesLoader). Always create a new configuration when we need to modify it. --- src/Common/Config/ConfigHelper.cpp | 29 +++++++++++++++++++++++++++++ src/Common/Config/ConfigHelper.h | 13 ++++++++----- src/Storages/StorageDictionary.cpp | 13 +++++++++---- src/Storages/StorageDictionary.h | 2 +- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/Common/Config/ConfigHelper.cpp b/src/Common/Config/ConfigHelper.cpp index 69fe42de86c..6de63fe78d7 100644 --- a/src/Common/Config/ConfigHelper.cpp +++ b/src/Common/Config/ConfigHelper.cpp @@ -1,5 +1,7 @@ #include #include +#include + namespace DB { @@ -7,6 +9,33 @@ namespace DB namespace ConfigHelper { +namespace +{ + void cloneImpl(Poco::Util::AbstractConfiguration & dest, const Poco::Util::AbstractConfiguration & src, const std::string & prefix = "") + { + std::vector keys; + src.keys(prefix, keys); + if (!keys.empty()) + { + std::string prefix_with_dot = prefix + "."; + for (const auto & key : keys) + cloneImpl(dest, src, prefix_with_dot + key); + } + else if (!prefix.empty()) + { + dest.setString(prefix, src.getRawString(prefix)); + } + } +} + + +Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src) +{ + Poco::AutoPtr res(new Poco::Util::XMLConfiguration()); + cloneImpl(*res, src); + return res; +} + bool getBool(const Poco::Util::AbstractConfiguration & config, const std::string & key, bool default_, bool empty_as) { if (!config.has(key)) diff --git a/src/Common/Config/ConfigHelper.h b/src/Common/Config/ConfigHelper.h index a1b630fa726..4f13bc4ad78 100644 --- a/src/Common/Config/ConfigHelper.h +++ b/src/Common/Config/ConfigHelper.h @@ -1,18 +1,21 @@ #pragma once +#include #include -namespace Poco + +namespace Poco::Util { - namespace Util - { - class AbstractConfiguration; - } + class AbstractConfiguration; } + namespace DB::ConfigHelper { +/// Clones a configuration. +Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src); + /// The behavior is like `config.getBool(key, default_)`, /// except when the tag is empty (aka. self-closing), `empty_as` will be used instead of throwing Poco::Exception. bool getBool(const Poco::Util::AbstractConfiguration & config, const std::string & key, bool default_ = false, bool empty_as = true); diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index ee76522fe48..414c66e9522 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -221,13 +222,17 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id) { std::lock_guard lock(dictionary_config_mutex); + auto new_configuration = ConfigHelper::clone(*configuration); + + new_configuration->setString("dictionary.database", new_table_id.database_name); + new_configuration->setString("dictionary.name", new_table_id.table_name); - configuration->setString("dictionary.database", new_table_id.database_name); - configuration->setString("dictionary.name", new_table_id.table_name); if (move_to_atomic) - configuration->setString("dictionary.uuid", toString(new_table_id.uuid)); + new_configuration->setString("dictionary.uuid", toString(new_table_id.uuid)); else if (move_to_ordinary) - configuration->remove("dictionary.uuid"); + new_configuration->remove("dictionary.uuid"); + + configuration = new_configuration; } const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 5c6b2f6d8ad..a414e4586d9 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -101,7 +101,7 @@ private: const Location location; mutable std::mutex dictionary_config_mutex; - LoadablesConfigurationPtr configuration; + LoadablesConfigurationPtr configuration TSA_GUARDED_BY(dictionary_config_mutex); scope_guard remove_repository_callback; From 5c60b1fab17823bd865394a54003336e5e0965a6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 15 Mar 2024 02:52:30 +0100 Subject: [PATCH 125/332] Remove unnecessary reloading while renaming a dictionary in Atomic database. --- .../AbstractConfigurationComparison.cpp | 91 ++++++++++++------- .../Config/AbstractConfigurationComparison.h | 6 ++ .../ExternalDictionariesLoader.cpp | 14 +++ src/Interpreters/ExternalDictionariesLoader.h | 3 + src/Interpreters/ExternalLoader.cpp | 18 +++- src/Interpreters/ExternalLoader.h | 4 + 6 files changed, 98 insertions(+), 38 deletions(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index c00a1efeebe..e983bd32f51 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -19,6 +19,56 @@ namespace result += subkey; return result; } + + bool isSameConfigurationImpl(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set * ignore_keys) + { + if (&left == &right && left_key == right_key) + return true; + + /// Get the subkeys of the left and right configurations. + Poco::Util::AbstractConfiguration::Keys left_subkeys; + Poco::Util::AbstractConfiguration::Keys right_subkeys; + left.keys(left_key, left_subkeys); + right.keys(right_key, right_subkeys); + + if (ignore_keys) + { + std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + } + + /// Check that the right configuration has the same set of subkeys as the left configuration. + if (left_subkeys.size() != right_subkeys.size()) + return false; + + if (left_subkeys.empty()) + { + if (left.hasProperty(left_key)) + { + return right.hasProperty(right_key) && (left.getRawString(left_key) == right.getRawString(right_key)); + } + else + { + return !right.hasProperty(right_key); + } + } + else + { + /// Go through all the subkeys and compare corresponding parts of the configurations. + std::unordered_set left_subkeys_set{left_subkeys.begin(), left_subkeys.end()}; + for (const auto & subkey : right_subkeys) + { + if (!left_subkeys_set.contains(subkey)) + return false; + + if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) + return false; + } + return true; + } + } } @@ -52,41 +102,14 @@ bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key) { - if (&left == &right && left_key == right_key) - return true; + return isSameConfigurationImpl(left, left_key, right, right_key, /* ignore_keys= */ nullptr); +} - bool has_property = left.hasProperty(left_key); - if (has_property != right.hasProperty(right_key)) - return false; - if (has_property) - { - /// The left and right configurations contains values so we can compare them. - if (left.getRawString(left_key) != right.getRawString(right_key)) - return false; - } - - /// Get the subkeys of the left and right configurations. - Poco::Util::AbstractConfiguration::Keys subkeys; - left.keys(left_key, subkeys); - - { - /// Check that the right configuration has the same set of subkeys as the left configuration. - Poco::Util::AbstractConfiguration::Keys right_subkeys; - right.keys(right_key, right_subkeys); - std::unordered_set left_subkeys{subkeys.begin(), subkeys.end()}; - if ((left_subkeys.size() != right_subkeys.size()) || (left_subkeys.size() != subkeys.size())) - return false; - for (const auto & right_subkey : right_subkeys) - if (!left_subkeys.contains(right_subkey)) - return false; - } - - /// Go through all the subkeys and compare corresponding parts of the configurations. - for (const auto & subkey : subkeys) - if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) - return false; - - return true; +bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys) +{ + return isSameConfigurationImpl(left, left_key, right, right_key, !ignore_keys.empty() ? &ignore_keys : nullptr); } } diff --git a/src/Common/Config/AbstractConfigurationComparison.h b/src/Common/Config/AbstractConfigurationComparison.h index 75e0b77b1ca..edaff8d5363 100644 --- a/src/Common/Config/AbstractConfigurationComparison.h +++ b/src/Common/Config/AbstractConfigurationComparison.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace Poco::Util { @@ -33,6 +34,11 @@ namespace DB bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key); + /// Returns true if specified subviews of the two configurations contains the same keys and values, but without checking specified keys. + bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys); + inline bool operator==(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right) { return isSameConfiguration(left, right); diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 760f94d18c2..0591a6232d0 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "config.h" @@ -41,6 +42,19 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, getContext(), created_from_ddl); } +bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, + const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const +{ + /// If the database is atomic then a dictionary can be renamed without reloading. + if (!old_config.getString(old_key_in_config + ".uuid", "").empty() && !new_config.getString(new_key_in_config + ".uuid", "").empty()) + { + static const std::unordered_set ignore_keys{"name", "database"}; + bool only_name_or_database_may_differ = isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); + return !only_name_or_database_may_differ; + } + return true; +} + void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const { IDictionary & dict = static_cast(object); diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index 3c226f998be..f3aa8fa5f8e 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -43,6 +43,9 @@ protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; + bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, + const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const override; + void updateObjectFromConfigWithoutReloading( IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const override; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 3e162b1114f..6ab083f9fb7 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -88,6 +88,7 @@ namespace } }; + using DoesConfigChangeRequiresReloadingObjectFunction = std::function; using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -402,10 +403,12 @@ public: LoadingDispatcher( const CreateObjectFunction & create_object_function_, + const DoesConfigChangeRequiresReloadingObjectFunction & does_config_change_requires_reloading_object_, const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, const String & type_name_, LoggerPtr log_) : create_object(create_object_function_) + , does_config_change_requires_reloading_object(does_config_change_requires_reloading_object_) , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) , type_name(type_name_) , log(log_) @@ -472,10 +475,14 @@ public: if (info.triedToLoad()) { - /// The object has been tried to load before, so it is currently in use or was in use - /// and we should try to reload it with the new config. - LOG_TRACE(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); - startLoading(info, true); + bool config_change_requires_reloading = does_config_change_requires_reloading_object(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + if (config_change_requires_reloading) + { + /// The object has been tried to load before, so it is currently in use or was in use + /// and we should try to reload it with the new config. + LOG_TRACE(log, "Will reload '{}' because its configuration has been changed and there were attempts to load it before", name); + startLoading(info, true); + } } } } @@ -1204,6 +1211,7 @@ private: } const CreateObjectFunction create_object; + const DoesConfigChangeRequiresReloadingObjectFunction does_config_change_requires_reloading_object; const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; LoggerPtr log; @@ -1290,6 +1298,8 @@ ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) , loading_dispatcher(std::make_unique( [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, + [this](const Poco::Util::AbstractConfiguration & config_1, const String & key_in_config_1, const Poco::Util::AbstractConfiguration & config_2, const String & key_in_config_2) + { return doesConfigChangeRequiresReloadingObject(config_1, key_in_config_1, config_2, key_in_config_2); }, [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, type_name_, diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index 279157b6d51..781d8ca9e7b 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -213,6 +213,10 @@ public: protected: virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + /// Returns whether the object must be reloaded after a specified change in its configuration. + virtual bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & /* old_config */, const String & /* old_key_in_config */, + const Poco::Util::AbstractConfiguration & /* new_config */, const String & /* new_key_in_config */) const { return true; /* always reload */ } + /// Updates the object from the configuration without reloading as much as possible. virtual void updateObjectFromConfigWithoutReloading( IExternalLoadable & /* object */, const Poco::Util::AbstractConfiguration & /* config */, const String & /* key_in_config */) const {} From 0240ffa5a6dab4f747d17295648d030af907f8f6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 19:31:52 +0100 Subject: [PATCH 126/332] Fetch the changed files only once --- tests/ci/pr_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 6f4b400f7a8..9bd30f3c58e 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -337,6 +337,9 @@ class PRInfo: ) def fetch_changed_files(self): + if self.changed_files_requested: + return + if not getattr(self, "diff_urls", False): raise TypeError("The event does not have diff URLs") From 49017fc163174e4cb4b3e9253094f399503add42 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 20:39:47 +0100 Subject: [PATCH 127/332] Run the style-check on top of the merge-commit of PRs --- tests/ci/style_check.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 7fb5d22a3ce..4daa6921d6b 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -67,26 +67,6 @@ def parse_args(): return parser.parse_args() -def checkout_head(pr_info: PRInfo) -> None: - # It works ONLY for PRs, and only over ssh, so either - # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work - assert pr_info.number - if not pr_info.head_name == pr_info.base_name: - # We can't push to forks, sorry folks - return - remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] - fetch_cmd = ( - f"{GIT_PREFIX} fetch --depth=1 " - f"{remote_url} {pr_info.head_ref}:head-{pr_info.head_ref}" - ) - if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): - with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): - git_runner(fetch_cmd) - else: - git_runner(fetch_cmd) - git_runner(f"git checkout -f head-{pr_info.head_ref}") - - def commit_push_staged(pr_info: PRInfo) -> None: # It works ONLY for PRs, and only over ssh, so either # ROBOT_CLICKHOUSE_SSH_KEY should be set or ssh-agent should work @@ -177,8 +157,6 @@ def main(): _ = future1.result() if run_pycheck: - if args.push: - checkout_head(pr_info) logging.info("Run py files check: %s", cmd_py) future2 = executor.submit(subprocess.run, cmd_py, shell=True) _ = future2.result() From c3f386d06a2c12f08d168accbf51457ad7a735c0 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 11 Mar 2024 20:57:35 +0100 Subject: [PATCH 128/332] Fix the autofixer --- tests/ci/style_check.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 4daa6921d6b..0332383df91 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -78,24 +78,23 @@ def commit_push_staged(pr_info: PRInfo) -> None: if not git_staged: return remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"] + head = git_runner("git rev-parse HEAD^{}") git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'") - push_cmd = ( - f"{GIT_PREFIX} push {remote_url} head-{pr_info.head_ref}:{pr_info.head_ref}" + # The fetch to avoid issue 'pushed branch tip is behind its remote' + fetch_cmd = ( + f"{GIT_PREFIX} fetch {remote_url} --no-recurse-submodules --depth=2 {head}" ) + push_cmd = f"{GIT_PREFIX} push {remote_url} HEAD:{pr_info.head_ref}" if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""): with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): + git_runner(fetch_cmd) git_runner(push_cmd) - else: - git_runner(push_cmd) + return + + git_runner(fetch_cmd) + git_runner(push_cmd) -def checkout_last_ref(pr_info: PRInfo) -> None: - # Checkout the merge commit back to avoid special effects - assert pr_info.number - if not pr_info.head_name == pr_info.base_name: - # We can't push to forks, sorry folks - return - git_runner("git checkout -f -") def main(): @@ -162,7 +161,6 @@ def main(): _ = future2.result() if args.push: commit_push_staged(pr_info) - checkout_last_ref(pr_info) subprocess.check_call( f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} " From 41432c56a8f13a8914b65b265d9855847314db79 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 10:50:44 +0100 Subject: [PATCH 129/332] Enable shellcheck conditionally --- tests/ci/style_check.py | 84 +++++++++++-------- utils/check-style/check_shell.sh | 13 +++ .../check-style/process_style_check_result.py | 4 +- 3 files changed, 64 insertions(+), 37 deletions(-) create mode 100755 utils/check-style/check_shell.sh diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 0332383df91..797c0ef12d0 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -8,8 +8,9 @@ import subprocess import sys from concurrent.futures import ProcessPoolExecutor from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, Union +import magic from docker_images_helper import get_docker_image, pull_image from env_helper import CI, REPO_COPY, TEMP_PATH from git_helper import GIT_PREFIX, git_runner @@ -95,6 +96,24 @@ def commit_push_staged(pr_info: PRInfo) -> None: git_runner(push_cmd) +def is_python(file: Union[Path, str]) -> bool: + """returns if the changed file in the repository is python script""" + # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, + # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-script.python" + ) + + +def is_shell(file: Union[Path, str]) -> bool: + """returns if the changed file in the repository is shell script""" + # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, + # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-shellscript" + ) def main(): @@ -111,29 +130,28 @@ def main(): temp_path.mkdir(parents=True, exist_ok=True) pr_info = PRInfo() + run_cpp_check = True + run_shell_check = True + run_python_check = True + if CI and pr_info.number > 0: + pr_info.fetch_changed_files() + run_cpp_check = not any( + is_python(file) or is_shell(file) for file in pr_info.changed_files + ) + run_shell_check = any(is_shell(file) for file in pr_info.changed_files) + run_python_check = any(is_python(file) for file in pr_info.changed_files) IMAGE_NAME = "clickhouse/style-test" image = pull_image(get_docker_image(IMAGE_NAME)) - cmd_cpp = ( + docker_command = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_cpp.sh" - ) - - cmd_py = ( - f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " - f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_py.sh" - ) - - cmd_docs = ( - f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " - f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"--entrypoint= -w/ClickHouse/utils/check-style " - f"{image} ./check_docs.sh" + f"--entrypoint= -w/ClickHouse/utils/check-style {image}" ) + cmd_docs = f"{docker_command} ./check_docs.sh" + cmd_cpp = f"{docker_command} ./check_cpp.sh" + cmd_py = f"{docker_command} ./check_py.sh" + cmd_shell = f"{docker_command} ./check_shell.sh" with ProcessPoolExecutor(max_workers=2) as executor: logging.info("Run docs files check: %s", cmd_docs) @@ -141,26 +159,22 @@ def main(): # Parallelization does not make it faster - run subsequently _ = future.result() - run_cppcheck = True - run_pycheck = True - if CI and pr_info.number > 0: - pr_info.fetch_changed_files() - if not any(file.endswith(".py") for file in pr_info.changed_files): - run_pycheck = False - if all(file.endswith(".py") for file in pr_info.changed_files): - run_cppcheck = False - - if run_cppcheck: + if run_cpp_check: logging.info("Run source files check: %s", cmd_cpp) - future1 = executor.submit(subprocess.run, cmd_cpp, shell=True) - _ = future1.result() + future = executor.submit(subprocess.run, cmd_cpp, shell=True) + _ = future.result() - if run_pycheck: + if run_python_check: logging.info("Run py files check: %s", cmd_py) - future2 = executor.submit(subprocess.run, cmd_py, shell=True) - _ = future2.result() - if args.push: - commit_push_staged(pr_info) + future = executor.submit(subprocess.run, cmd_py, shell=True) + _ = future.result() + if run_shell_check: + logging.info("Run shellcheck check: %s", cmd_shell) + future = executor.submit(subprocess.run, cmd_shell, shell=True) + _ = future.result() + + if args.push: + commit_push_staged(pr_info) subprocess.check_call( f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} " diff --git a/utils/check-style/check_shell.sh b/utils/check-style/check_shell.sh new file mode 100755 index 00000000000..94f06220f88 --- /dev/null +++ b/utils/check-style/check_shell.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv + +start_total=$(date +%s) + +start=$(date +%s) +./shellcheck-run.sh |& tee /test_output/shellcheck.txt +runtime=$(($(date +%s)-start)) +echo "Check shellcheck. Done. $runtime seconds." + +runtime=$(($(date +%s)-start_total)) +echo "Check style total. Done. $runtime seconds." diff --git a/utils/check-style/process_style_check_result.py b/utils/check-style/process_style_check_result.py index 8c9837b4fc6..f1ca53eba3b 100755 --- a/utils/check-style/process_style_check_result.py +++ b/utils/check-style/process_style_check_result.py @@ -13,8 +13,8 @@ def process_result(result_folder): description = "" test_results = [] checks = ( - # "duplicate includes", - # "shellcheck", + "duplicate includes", + "shellcheck", "style", "pylint", "black", From 0e6b5705fbdce4c7032d39dbdae452f266e1b0b3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 12:20:30 +0100 Subject: [PATCH 130/332] Install python-magic in style-check, split pip and apt --- docker/test/style/Dockerfile | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index b4ffcfb597c..5d53d03606f 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -8,20 +8,22 @@ ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - aspell \ - curl \ - git \ - file \ - libxml2-utils \ - moreutils \ - python3-fuzzywuzzy \ - python3-pip \ - yamllint \ - locales \ - && pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ - requests types-requests \ + aspell \ + curl \ + git \ + file \ + libxml2-utils \ + moreutils \ + python3-fuzzywuzzy \ + python3-pip \ + yamllint \ + locales \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \ + && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* + +# python-magic is the same version as in Ubuntu 22.04 +RUN pip3 install black==23.12.0 boto3 codespell==2.2.1 mypy==1.8.0 PyGithub unidiff pylint==3.1.0 \ + python-magic==0.4.24 requests types-requests \ && rm -rf /root/.cache/pip RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8 From 97b0513f9928de5f0207978d8138b478bc1677bd Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 12 Mar 2024 19:21:05 +0100 Subject: [PATCH 131/332] Enable duplicate includes back, rewrite and parallelize them --- utils/check-style/check-duplicate-includes.sh | 5 ++- utils/check-style/check_cpp.sh | 31 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/utils/check-style/check-duplicate-includes.sh b/utils/check-style/check-duplicate-includes.sh index 64aca4d180d..ca8b98d9537 100755 --- a/utils/check-style/check-duplicate-includes.sh +++ b/utils/check-style/check-duplicate-includes.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash ROOT_PATH=$(git rev-parse --show-toplevel) +NPROC=$(($(nproc) + 3)) # Find duplicate include directives -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | while read file; do grep -P '^#include ' $file | sort | uniq -c | grep -v -P '^\s+1\s' && echo $file; done | sed '/^[[:space:]]*$/d' +find "$ROOT_PATH"/{src,base,programs,utils} -type f '(' -name '*.h' -or -name '*.cpp' ')' -print0 | \ + xargs -0 -I {} -P "$NPROC" bash -c 'grep -P "^#include " "{}" | sort | uniq -c | grep -v -P "^\s+1\s" && echo "{}" '| \ + sed '/^\s*$/d' diff --git a/utils/check-style/check_cpp.sh b/utils/check-style/check_cpp.sh index ea90d79418c..dc1dae70e22 100755 --- a/utils/check-style/check_cpp.sh +++ b/utils/check-style/check_cpp.sh @@ -4,35 +4,32 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv -start_total=`date +%s` +start_total=$(date +%s) -# FIXME: 30 sec to wait -# echo "Check duplicates" | ts -# ./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +start=$(date +%s) +./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +runtime=$(($(date +%s)-start)) +echo "Check duplicates. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-style -n |& tee /test_output/style_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check style. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check whitespaces. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-workflows |& tee /test_output/workflows_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check workflows. Done. $runtime seconds." -start=`date +%s` +start=$(date +%s) ./check-submodules |& tee /test_output/submodules_output.txt -runtime=$((`date +%s`-start)) +runtime=$(($(date +%s)-start)) echo "Check submodules. Done. $runtime seconds." -# FIXME: 6 min to wait -# echo "Check shell scripts with shellcheck" | ts -# ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt - -runtime=$((`date +%s`-start_total)) +runtime=$(($(date +%s)-start_total)) echo "Check style total. Done. $runtime seconds." From a5331ea54462c5cdb300c7f48d28d8795fac1bc9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 Mar 2024 13:49:05 +0100 Subject: [PATCH 132/332] Rewrite list-licenses.sh to use array+find --- utils/list-licenses/list-licenses.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/list-licenses/list-licenses.sh b/utils/list-licenses/list-licenses.sh index f09168a0596..cc730464e8e 100755 --- a/utils/list-licenses/list-licenses.sh +++ b/utils/list-licenses/list-licenses.sh @@ -5,15 +5,16 @@ if [[ "$OSTYPE" == "darwin"* ]]; then GREP_CMD=ggrep FIND_CMD=gfind else - FIND_CMD=find - GREP_CMD=grep + FIND_CMD='find' + GREP_CMD='grep' fi ROOT_PATH="$(git rev-parse --show-toplevel)" LIBS_PATH="${ROOT_PATH}/contrib" -ls -1 -d ${LIBS_PATH}/*/ "${ROOT_PATH}/base/poco" | ${GREP_CMD} -F -v -- '-cmake' | LC_ALL=C sort | while read LIB; do - LIB_NAME=$(basename $LIB) +mapfile -t libs < <(echo "${ROOT_PATH}/base/poco"; find "${LIBS_PATH}" -type d -maxdepth 1 ! -name '*-cmake' | LC_ALL=C sort) +for LIB in "${libs[@]}"; do + LIB_NAME=$(basename "$LIB") LIB_LICENSE=$( LC_ALL=C ${FIND_CMD} "$LIB" -type f -and '(' -iname 'LICENSE*' -or -iname 'COPYING*' -or -iname 'COPYRIGHT*' ')' -and -not '(' -iname '*.html' -or -iname '*.htm' -or -iname '*.rtf' -or -name '*.cpp' -or -name '*.h' -or -iname '*.json' ')' -printf "%d\t%p\n" | From 070f7bee9b550962cba6768ffd3a74306e4e2107 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 14 Mar 2024 13:56:43 +0100 Subject: [PATCH 133/332] Fix shellcheck issues in durability-test.sh --- utils/durability-test/durability-test.sh | 51 +++++++++++------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/utils/durability-test/durability-test.sh b/utils/durability-test/durability-test.sh index 97c39473b69..188ef3452e7 100755 --- a/utils/durability-test/durability-test.sh +++ b/utils/durability-test/durability-test.sh @@ -15,7 +15,6 @@ Usage: URL=http://cloud-images.ubuntu.com/bionic/current IMAGE=bionic-server-cloudimg-amd64.img SSH_PORT=11022 -CLICKHOUSE_PORT=9090 PASSWORD=root TABLE_NAME=$1 @@ -34,19 +33,18 @@ function run() function copy() { - sshpass -p $PASSWORD scp -r -P $SSH_PORT $1 root@localhost:$2 2>/dev/null + sshpass -p $PASSWORD scp -r -P $SSH_PORT "$1" "root@localhost:$2" 2>/dev/null } function wait_vm_for_start() { echo "Waiting until VM started..." started=0 - for i in {0..100}; do - run "exit" - if [ $? -eq 0 ]; then + for _ in {0..100}; do + if run "exit"; then started=1 break - fi + fi sleep 1s done @@ -62,9 +60,8 @@ function wait_clickhouse_for_start() { echo "Waiting until ClickHouse started..." started=0 - for i in {0..30}; do - run "clickhouse client --query 'select 1'" > /dev/null - if [ $? -eq 0 ]; then + for _ in {0..30}; do + if run "clickhouse client --query 'select 1'" > /dev/null; then started=1 break fi @@ -105,11 +102,11 @@ if [[ -z $CLICKHOUSE_CONFIG_DIR ]]; then CLICKHOUSE_CONFIG_DIR=/etc/clickhouse-server fi -echo "Using ClickHouse binary:" $CLICKHOUSE_BINARY -echo "Using ClickHouse config from:" $CLICKHOUSE_CONFIG_DIR +echo "Using ClickHouse binary: $CLICKHOUSE_BINARY" +echo "Using ClickHouse config from: $CLICKHOUSE_CONFIG_DIR" -copy $CLICKHOUSE_BINARY /usr/bin -copy $CLICKHOUSE_CONFIG_DIR /etc +copy "$CLICKHOUSE_BINARY" /usr/bin +copy "$CLICKHOUSE_CONFIG_DIR" /etc run "mv /etc/$CLICKHOUSE_CONFIG_DIR /etc/clickhouse-server" echo "Prepared VM" @@ -118,23 +115,23 @@ echo "Starting ClickHouse" run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" & wait_clickhouse_for_start -query=`cat $CREATE_QUERY` -echo "Executing query:" $query +query=$(cat "$CREATE_QUERY") +echo "Executing query: $query" run "clickhouse client --query '$query'" -query=`cat $INSERT_QUERY` -echo "Will run in a loop query: " $query +query=$(cat "$INSERT_QUERY") +echo "Will run in a loop query: $query" run "clickhouse benchmark <<< '$query' -c 8" & echo "Running queries" -pid=`pidof qemu-system-x86_64` +pid=$(pidof qemu-system-x86_64) sec=$(( (RANDOM % 5) + 25 )) ms=$(( RANDOM % 1000 )) echo "Will kill VM in $sec.$ms sec" sleep $sec.$ms -kill -9 $pid +kill -9 "$pid" echo "Restarting" @@ -147,22 +144,22 @@ run "rm -r *data/system" run "clickhouse server --config-file=/etc/clickhouse-server/config.xml > clickhouse-server.log 2>&1" & wait_clickhouse_for_start -pid=`pidof qemu-system-x86_64` -result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'"` +pid=$(pidof qemu-system-x86_64) +result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Caught exception while loading metadata'") if [[ -n $result ]]; then echo "FAIL. Can't attach table:" - echo $result - kill -9 $pid + echo "$result" + kill -9 "$pid" exit 1 fi -result=`run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'"` +result=$(run "grep $TABLE_NAME clickhouse-server.log | grep 'Considering to remove broken part'") if [[ -n $result ]]; then echo "FAIL. Have broken parts:" - echo $result - kill -9 $pid + echo "$result" + kill -9 "$pid" exit 1 fi -kill -9 $pid +kill -9 "$pid" echo OK From 679bf92cd146f0855ba5e02bc0c343c46506b534 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 15:08:46 +0000 Subject: [PATCH 134/332] Split test 00159_parallel_formatting_json_and_friends to avoid timeouts --- ...l_formatting_json_and_friends_1.reference} | 12 ----------- ...parallel_formatting_json_and_friends_1.sh} | 2 +- ...el_formatting_json_and_friends_2.reference | 12 +++++++++++ ..._parallel_formatting_json_and_friends_2.sh | 20 +++++++++++++++++++ 4 files changed, 33 insertions(+), 13 deletions(-) rename tests/queries/1_stateful/{00159_parallel_formatting_json_and_friends.reference => 00159_parallel_formatting_json_and_friends_1.reference} (51%) rename tests/queries/1_stateful/{00159_parallel_formatting_json_and_friends.sh => 00159_parallel_formatting_json_and_friends_1.sh} (83%) create mode 100644 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference create mode 100755 tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference similarity index 51% rename from tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference rename to tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference index 787b57ce761..cd425198651 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.reference @@ -14,15 +14,3 @@ JSONCompactEachRowWithNames, false e3231b1c8187de4da6752d692b2ddba9 - JSONCompactEachRowWithNames, true e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactStringsEachRowWithNames, false -e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactStringsEachRowWithNames, true -e3231b1c8187de4da6752d692b2ddba9 - -JSONCompactEachRowWithNamesAndTypes, false -d40c4327c63eded184eee185a5330e12 - -JSONCompactEachRowWithNamesAndTypes, true -d40c4327c63eded184eee185a5330e12 - -JSONCompactStringsEachRowWithNamesAndTypes, false -d40c4327c63eded184eee185a5330e12 - -JSONCompactStringsEachRowWithNamesAndTypes, true -d40c4327c63eded184eee185a5330e12 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh similarity index 83% rename from tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh rename to tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh index 77dd330099a..0cc4636ef5d 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -FORMATS=('JSONEachRow' 'JSONCompactEachRow' 'JSONCompactStringsEachRow' 'JSONCompactEachRowWithNames' 'JSONCompactStringsEachRowWithNames' 'JSONCompactEachRowWithNamesAndTypes' 'JSONCompactStringsEachRowWithNamesAndTypes') +FORMATS=('JSONEachRow' 'JSONCompactEachRow' 'JSONCompactStringsEachRow' 'JSONCompactEachRowWithNames') for format in "${FORMATS[@]}" do diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference new file mode 100644 index 00000000000..ba7a8ac5fdc --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.reference @@ -0,0 +1,12 @@ +JSONCompactStringsEachRowWithNames, false +e3231b1c8187de4da6752d692b2ddba9 - +JSONCompactStringsEachRowWithNames, true +e3231b1c8187de4da6752d692b2ddba9 - +JSONCompactEachRowWithNamesAndTypes, false +d40c4327c63eded184eee185a5330e12 - +JSONCompactEachRowWithNamesAndTypes, true +d40c4327c63eded184eee185a5330e12 - +JSONCompactStringsEachRowWithNamesAndTypes, false +d40c4327c63eded184eee185a5330e12 - +JSONCompactStringsEachRowWithNamesAndTypes, true +d40c4327c63eded184eee185a5330e12 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh new file mode 100755 index 00000000000..1c49305fe53 --- /dev/null +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends_2.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +FORMATS=('JSONCompactStringsEachRowWithNames' 'JSONCompactEachRowWithNamesAndTypes' 'JSONCompactStringsEachRowWithNamesAndTypes') + +for format in "${FORMATS[@]}" +do + echo "$format, false"; + $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum + + echo "$format, true"; + $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=true -q \ + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum +done From 514faa851c32d26620fb3a086f3dbfe40ecd1a23 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 18 Mar 2024 15:26:01 +0000 Subject: [PATCH 135/332] Make test 00159_parallel_formatting_tsv_and_friends.sh faster --- .../1_stateful/00159_parallel_formatting_tsv_and_friends.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh index 02441190b91..393a9fc4282 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh @@ -11,9 +11,9 @@ for format in "${FORMATS[@]}" do echo "$format, false"; $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum echo "$format, true"; $CLICKHOUSE_CLIENT --max_threads=0 --output_format_parallel_formatting=true -q \ - "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum done From efe5373f2bae7c0101862c7b4e59874d6a332271 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 18 Mar 2024 15:54:18 +0000 Subject: [PATCH 136/332] refactoring of reading from compact parts --- .../MergeTree/MergeTreeDataPartCompact.cpp | 23 +- src/Storages/MergeTree/MergeTreeIOSettings.h | 13 + .../MergeTree/MergeTreeIndexReader.cpp | 21 +- .../MergeTree/MergeTreeMarksLoader.cpp | 78 +-- src/Storages/MergeTree/MergeTreeMarksLoader.h | 51 +- .../MergeTree/MergeTreeReaderCompact.cpp | 515 ++++-------------- .../MergeTree/MergeTreeReaderCompact.h | 77 +-- .../MergeTreeReaderCompactSingleBuffer.cpp | 108 ++++ .../MergeTreeReaderCompactSingleBuffer.h | 33 ++ .../MergeTree/MergeTreeReaderStream.cpp | 441 ++++++++++----- .../MergeTree/MergeTreeReaderStream.h | 136 ++++- .../MergeTree/MergeTreeReaderWide.cpp | 27 +- src/Storages/StorageMergeTreeIndex.cpp | 4 +- 13 files changed, 814 insertions(+), 713 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp create mode 100644 src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 9f201ab3b81..77a73fcc8f9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -1,10 +1,8 @@ #include "MergeTreeDataPartCompact.h" #include -#include +#include #include -#include #include -#include namespace DB @@ -41,21 +39,12 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { auto read_info = std::make_shared(shared_from_this(), alter_conversions); - auto * load_marks_threadpool - = reader_settings.read_settings.load_marks_asynchronously ? &read_info->getContext()->getLoadMarksThreadpool() : nullptr; - return std::make_unique( - read_info, - columns_to_read, - virtual_fields, - storage_snapshot, - uncompressed_cache, - mark_cache, - mark_ranges, - reader_settings, - load_marks_threadpool, - avg_value_size_hints, - profile_callback); + return std::make_unique( + read_info, columns_to_read, virtual_fields, + storage_snapshot, uncompressed_cache, + mark_cache, mark_ranges, reader_settings, + avg_value_size_hints, profile_callback, CLOCK_MONOTONIC_COARSE); } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 379d7af8569..feeb1808a6f 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -13,6 +13,11 @@ namespace DB class MMappedFileCache; using MMappedFileCachePtr = std::shared_ptr; +enum class CompactPartsReadMethod +{ + SingleBuffer, + MultiBuffer, +}; struct MergeTreeReaderSettings { @@ -25,12 +30,20 @@ struct MergeTreeReaderSettings bool checksum_on_read = true; /// True if we read in order of sorting key. bool read_in_order = false; + /// Use one buffer for each column or for all columns while reading from compact. + CompactPartsReadMethod compact_parts_read_method = CompactPartsReadMethod::SingleBuffer; + /// True if we read stream for dictionary of LowCardinality type. + bool is_low_cardinality_dictionary = false; + /// True if data may be compressed by different codecs in one stream. + bool allow_different_codecs = false; /// Deleted mask is applied to all reads except internal select from mutate some part columns. bool apply_deleted_mask = true; /// Put reading task in a common I/O pool, return Async state on prepare() bool use_asynchronous_read_from_pool = false; /// If PREWHERE has multiple conditions combined with AND, execute them in separate read/filtering steps. bool enable_multiple_prewhere_read_steps = false; + /// If true, try to lower size of read buffer according to granule size and compressed block size. + bool adjust_read_buffer_size = true; }; struct MergeTreeWriterSettings diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index f0b1aacc239..6012994b46d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -20,14 +20,23 @@ std::unique_ptr makeIndexReader( auto context = part->storage.getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; - return std::make_unique( + auto marks_loader = std::make_shared( std::make_shared(part, std::make_shared()), + mark_cache, + part->index_granularity_info.getMarksFilePath(index->getFileName()), + marks_count, + part->index_granularity_info, + settings.save_marks_in_cache, + settings.read_settings, + load_marks_threadpool, + /*num_columns_in_mark=*/ 1); + + return std::make_unique( + part->getDataPartStoragePtr(), index->getFileName(), extension, marks_count, - all_mark_ranges, - std::move(settings), mark_cache, uncompressed_cache, - part->getFileSizeOrZero(index->getFileName() + extension), - &part->index_granularity_info, - ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE, false, load_marks_threadpool); + all_mark_ranges, std::move(settings), uncompressed_cache, + part->getFileSizeOrZero(index->getFileName() + extension), std::move(marks_loader), + ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE); } } diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 8250050412f..a7a635f3b72 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -28,6 +28,23 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +MergeTreeMarksGetter::MergeTreeMarksGetter(MarkCache::MappedPtr marks_, size_t num_columns_in_mark_) + : marks(std::move(marks_)), num_columns_in_mark(num_columns_in_mark_) +{ + assert(marks); +} + +MarkInCompressedFile MergeTreeMarksGetter::getMark(size_t row_index, size_t column_index) const +{ +#ifndef NDEBUG + if (column_index >= num_columns_in_mark) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Column index: {} is out of range [0, {})", column_index, num_columns_in_mark); +#endif + + return marks->get(row_index * num_columns_in_mark + column_index); +} + MergeTreeMarksLoader::MergeTreeMarksLoader( MergeTreeDataPartInfoForReaderPtr data_part_reader_, MarkCache * mark_cache_, @@ -37,58 +54,49 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( bool save_marks_in_cache_, const ReadSettings & read_settings_, ThreadPool * load_marks_threadpool_, - size_t columns_in_mark_) + size_t num_columns_in_mark_) : data_part_reader(data_part_reader_) , mark_cache(mark_cache_) , mrk_path(mrk_path_) , marks_count(marks_count_) , index_granularity_info(index_granularity_info_) , save_marks_in_cache(save_marks_in_cache_) - , columns_in_mark(columns_in_mark_) , read_settings(read_settings_) + , num_columns_in_mark(num_columns_in_mark_) , load_marks_threadpool(load_marks_threadpool_) { if (load_marks_threadpool) - { future = loadMarksAsync(); - } } MergeTreeMarksLoader::~MergeTreeMarksLoader() { if (future.valid()) - { future.wait(); - } } - -MarkInCompressedFile MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) +MergeTreeMarksGetterPtr MergeTreeMarksLoader::loadMarks() { - if (!marks) + std::lock_guard lock(load_mutex); + + if (marks) + return std::make_unique(marks, num_columns_in_mark); + + Stopwatch watch(CLOCK_MONOTONIC); + + if (future.valid()) { - Stopwatch watch(CLOCK_MONOTONIC); - - if (future.valid()) - { - marks = future.get(); - future = {}; - } - else - { - marks = loadMarks(); - } - - watch.stop(); - ProfileEvents::increment(ProfileEvents::WaitMarksLoadMicroseconds, watch.elapsedMicroseconds()); + marks = future.get(); + future = {}; + } + else + { + marks = loadMarksSync(); } -#ifndef NDEBUG - if (column_index >= columns_in_mark) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Column index: {} is out of range [0, {})", column_index, columns_in_mark); -#endif - - return marks->get(row_index * columns_in_mark + column_index); + watch.stop(); + ProfileEvents::increment(ProfileEvents::WaitMarksLoadMicroseconds, watch.elapsedMicroseconds()); + return std::make_unique(marks, num_columns_in_mark); } @@ -100,12 +108,12 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto data_part_storage = data_part_reader->getDataPartStorage(); size_t file_size = data_part_storage->getFileSize(mrk_path); - size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); + size_t mark_size = index_granularity_info.getMarkSizeInBytes(num_columns_in_mark); size_t expected_uncompressed_size = mark_size * marks_count; // We first read the marks into a temporary simple array, then compress them into a more compact // representation. - PODArray plain_marks(marks_count * columns_in_mark); // temporary + PODArray plain_marks(marks_count * num_columns_in_mark); // temporary auto full_mark_path = std::string(fs::path(data_part_storage->getFullPath()) / mrk_path); if (file_size == 0 && marks_count != 0) @@ -159,7 +167,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() size_t granularity; reader->readStrict( - reinterpret_cast(plain_marks.data() + i * columns_in_mark), columns_in_mark * sizeof(MarkInCompressedFile)); + reinterpret_cast(plain_marks.data() + i * num_columns_in_mark), num_columns_in_mark * sizeof(MarkInCompressedFile)); readBinaryLittleEndian(granularity, *reader); } @@ -182,13 +190,13 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() auto res = std::make_shared(plain_marks); - ProfileEvents::increment(ProfileEvents::LoadedMarksCount, marks_count * columns_in_mark); + ProfileEvents::increment(ProfileEvents::LoadedMarksCount, marks_count * num_columns_in_mark); ProfileEvents::increment(ProfileEvents::LoadedMarksMemoryBytes, res->approximateMemoryUsage()); return res; } -MarkCache::MappedPtr MergeTreeMarksLoader::loadMarks() +MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksSync() { MarkCache::MappedPtr loaded_marks; @@ -227,7 +235,7 @@ std::future MergeTreeMarksLoader::loadMarksAsync() [this]() -> MarkCache::MappedPtr { ProfileEvents::increment(ProfileEvents::BackgroundLoadingMarksTasks); - return loadMarks(); + return loadMarksSync(); }, *load_marks_threadpool, "LoadMarksThread"); diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 0889da0cb85..73dd462f2fa 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -10,13 +10,33 @@ namespace DB { struct MergeTreeIndexGranularityInfo; +using MarksPtr = MarkCache::MappedPtr; class Threadpool; +/// Class that helps to get marks by indexes. +/// Always immutable and thread safe. +/// Marks can be shared between several threads +/// that read columns from the same file. +class MergeTreeMarksGetter +{ +public: + MergeTreeMarksGetter(MarkCache::MappedPtr marks_, size_t num_columns_in_mark_); + + MarkInCompressedFile getMark(size_t row_index, size_t column_index) const; + size_t getNumColumns() const { return num_columns_in_mark; } + +private: + const MarkCache::MappedPtr marks; + const size_t num_columns_in_mark; +}; + +using MergeTreeMarksGetterPtr = std::unique_ptr; + +/// Class that helps to load marks on demand. +/// Thread safe, but locks while loading marks. class MergeTreeMarksLoader { public: - using MarksPtr = MarkCache::MappedPtr; - MergeTreeMarksLoader( MergeTreeDataPartInfoForReaderPtr data_part_reader_, MarkCache * mark_cache_, @@ -26,24 +46,27 @@ public: bool save_marks_in_cache_, const ReadSettings & read_settings_, ThreadPool * load_marks_threadpool_, - size_t columns_in_mark_ = 1); + size_t num_columns_in_mark_); ~MergeTreeMarksLoader(); - MarkInCompressedFile getMark(size_t row_index, size_t column_index = 0); + MergeTreeMarksGetterPtr loadMarks(); + size_t getNumColumns() const { return num_columns_in_mark; } private: - MergeTreeDataPartInfoForReaderPtr data_part_reader; - MarkCache * mark_cache = nullptr; - String mrk_path; - size_t marks_count; + const MergeTreeDataPartInfoForReaderPtr data_part_reader; + MarkCache * const mark_cache; + const String mrk_path; + const size_t marks_count; const MergeTreeIndexGranularityInfo & index_granularity_info; - bool save_marks_in_cache = false; - size_t columns_in_mark; - MarkCache::MappedPtr marks; - ReadSettings read_settings; + const bool save_marks_in_cache; + const ReadSettings read_settings; + const size_t num_columns_in_mark; - MarkCache::MappedPtr loadMarks(); + std::mutex load_mutex; + MarkCache::MappedPtr marks; + + MarkCache::MappedPtr loadMarksSync(); std::future loadMarksAsync(); MarkCache::MappedPtr loadMarksImpl(); @@ -51,4 +74,6 @@ private: ThreadPool * load_marks_threadpool; }; +using MergeTreeMarksLoaderPtr = std::shared_ptr; + } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 63824366722..3835c1e2054 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -10,10 +10,8 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_READ_ALL_DATA; - extern const int ARGUMENT_OUT_OF_BOUND; } - MergeTreeReaderCompact::MergeTreeReaderCompact( MergeTreeDataPartInfoForReaderPtr data_part_info_for_read_, NamesAndTypesList columns_, @@ -23,7 +21,6 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, - ThreadPool * load_marks_threadpool_, ValueSizeMap avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) @@ -37,91 +34,22 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( mark_ranges_, settings_, avg_value_size_hints_) - , marks_loader( - data_part_info_for_read_, - mark_cache, - data_part_info_for_read_->getIndexGranularityInfo().getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), - data_part_info_for_read_->getMarksCount(), - data_part_info_for_read_->getIndexGranularityInfo(), - settings.save_marks_in_cache, - settings.read_settings, - load_marks_threadpool_, - data_part_info_for_read_->getColumns().size()) + , marks_loader(std::make_shared( + data_part_info_for_read_, + mark_cache, + data_part_info_for_read_->getIndexGranularityInfo().getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), + data_part_info_for_read_->getMarksCount(), + data_part_info_for_read_->getIndexGranularityInfo(), + settings.save_marks_in_cache, + settings.read_settings, + settings_.read_settings.load_marks_asynchronously + ? &data_part_info_for_read_->getContext()->getLoadMarksThreadpool() : nullptr, + data_part_info_for_read_->getColumns().size())) , profile_callback(profile_callback_) , clock_type(clock_type_) { } -void MergeTreeReaderCompact::initialize() -{ - try - { - fillColumnPositions(); - - /// Do not use max_read_buffer_size, but try to lower buffer size with maximal size of granule to avoid reading much data. - auto buffer_size = getReadBufferSize(*data_part_info_for_read, marks_loader, column_positions, all_mark_ranges); - if (buffer_size) - settings.read_settings = settings.read_settings.adjustBufferSize(buffer_size); - - if (!settings.read_settings.local_fs_buffer_size || !settings.read_settings.remote_fs_buffer_size) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read to empty buffer."); - - const String path = MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; - auto data_part_storage = data_part_info_for_read->getDataPartStorage(); - - if (uncompressed_cache) - { - auto buffer = std::make_unique( - std::string(fs::path(data_part_storage->getFullPath()) / path), - [this, path, data_part_storage]() - { - return data_part_storage->readFile( - path, - settings.read_settings, - std::nullopt, std::nullopt); - }, - uncompressed_cache, - /* allow_different_codecs = */ true); - - if (profile_callback) - buffer->setProfileCallback(profile_callback, clock_type); - - if (!settings.checksum_on_read) - buffer->disableChecksumming(); - - cached_buffer = std::move(buffer); - data_buffer = cached_buffer.get(); - compressed_data_buffer = cached_buffer.get(); - } - else - { - auto buffer = - std::make_unique( - data_part_storage->readFile( - path, - settings.read_settings, - std::nullopt, std::nullopt), - /* allow_different_codecs = */ true); - - if (profile_callback) - buffer->setProfileCallback(profile_callback, clock_type); - - if (!settings.checksum_on_read) - buffer->disableChecksumming(); - - non_cached_buffer = std::move(buffer); - data_buffer = non_cached_buffer.get(); - compressed_data_buffer = non_cached_buffer.get(); - } - } - catch (...) - { - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - throw; - } -} - void MergeTreeReaderCompact::fillColumnPositions() { size_t columns_num = columns_to_read.size(); @@ -191,189 +119,31 @@ void MergeTreeReaderCompact::fillColumnPositions() } } -size_t MergeTreeReaderCompact::readRows( - size_t from_mark, size_t current_task_last_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) -{ - if (!initialized) - { - initialize(); - initialized = true; - } - - if (continue_reading) - from_mark = next_mark; - - size_t read_rows = 0; - size_t num_columns = columns_to_read.size(); - checkNumberOfColumns(num_columns); - - MutableColumns mutable_columns(num_columns); - for (size_t i = 0; i < num_columns; ++i) - { - if (column_positions[i] && res_columns[i] == nullptr) - res_columns[i] = columns_to_read[i].type->createColumn(*serializations[i]); - } - - while (read_rows < max_rows_to_read) - { - size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); - - /// If we need to read multiple subcolumns from a single column in storage, - /// we will read it this column only once and then reuse to extract all subcolumns. - std::unordered_map columns_cache_for_subcolumns; - - for (size_t pos = 0; pos < num_columns; ++pos) - { - if (!res_columns[pos]) - continue; - - try - { - auto & column = res_columns[pos]; - size_t column_size_before_reading = column->size(); - - readData(columns_to_read[pos], column, from_mark, current_task_last_mark, *column_positions[pos], rows_to_read, columns_for_offsets[pos], columns_cache_for_subcolumns); - - size_t read_rows_in_column = column->size() - column_size_before_reading; - if (read_rows_in_column != rows_to_read) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, - "Cannot read all data in MergeTreeReaderCompact. Rows read: {}. Rows expected: {}.", - read_rows_in_column, rows_to_read); - } - catch (...) - { - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - - /// Better diagnostics. - try - { - rethrow_exception(std::current_exception()); - } - catch (Exception & e) - { - e.addMessage(getMessageForDiagnosticOfBrokenPart(from_mark, max_rows_to_read)); - } - - throw; - } - } - - ++from_mark; - read_rows += rows_to_read; - } - - next_mark = from_mark; - - return read_rows; -} - void MergeTreeReaderCompact::readData( - const NameAndTypePair & name_and_type, ColumnPtr & column, - size_t from_mark, size_t current_task_last_mark, size_t column_position, size_t rows_to_read, - ColumnNameLevel name_level_for_offsets, std::unordered_map & columns_cache_for_subcolumns) + const NameAndTypePair & name_and_type, + ColumnPtr & column, + size_t rows_to_read, + const InputStreamGetter & getter) { - const auto & [name, type] = name_and_type; - std::optional column_for_offsets; - - if (name_level_for_offsets.has_value()) + try { - const auto & part_columns = data_part_info_for_read->getColumnsDescription(); - column_for_offsets = part_columns.getPhysical(name_level_for_offsets->first); - } + const auto [name, type] = name_and_type; + size_t column_size_before_reading = column->size(); - adjustUpperBound(current_task_last_mark); /// Must go before seek. + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; + deserialize_settings.getter = getter; + deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; - if (!isContinuousReading(from_mark, column_position)) - seekToMark(from_mark, column_position); - - /// If we read only offsets we have to read prefix anyway - /// to preserve correctness of serialization. - auto buffer_getter_for_prefix = [&](const auto &) -> ReadBuffer * - { - return data_buffer; - }; - - auto buffer_getter = [&](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * - { - /// Offset stream from another column could be read, in case of current - /// column does not exists (see findColumnForOffsets() in - /// MergeTreeReaderCompact::fillColumnPositions()) - if (name_level_for_offsets.has_value()) + if (name_and_type.isSubcolumn()) { - bool is_offsets = !substream_path.empty() && substream_path.back().type == ISerialization::Substream::ArraySizes; - if (!is_offsets) - return nullptr; + const auto & type_in_storage = name_and_type.getTypeInStorage(); + const auto & name_in_storage = name_and_type.getNameInStorage(); - /// Offset stream can be read only from columns of current level or - /// below (since it is OK to read all parent streams from the - /// alternative). - /// - /// Consider the following columns in nested "root": - /// - root.array Array(UInt8) - exists - /// - root.nested_array Array(Array(UInt8)) - does not exists (only_offsets_level=1) - /// - /// For root.nested_array it will try to read multiple streams: - /// - offsets (substream_path = {ArraySizes}) - /// OK - /// - root.nested_array elements (substream_path = {ArrayElements, ArraySizes}) - /// NOT OK - cannot use root.array offsets stream for this - /// - /// Here only_offsets_level is the level of the alternative stream, - /// and substream_path.size() is the level of the current stream. - if (name_level_for_offsets->second < ISerialization::getArrayLevel(substream_path)) - return nullptr; - } + auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); + ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - return data_buffer; - }; - - ISerialization::DeserializeBinaryBulkStatePtr state; - ISerialization::DeserializeBinaryBulkStatePtr state_for_prefix; - - ISerialization::DeserializeBinaryBulkSettings deserialize_settings; - deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; - bool columns_cache_was_used = false; - - if (name_and_type.isSubcolumn()) - { - NameAndTypePair name_type_in_storage{name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}; - ColumnPtr temp_column; - - auto it = columns_cache_for_subcolumns.find(name_type_in_storage.name); - if (!column_for_offsets && it != columns_cache_for_subcolumns.end()) - { - temp_column = it->second; - auto subcolumn = name_type_in_storage.type->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); - if (column->empty()) - column = IColumn::mutate(subcolumn); - else - column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); - - columns_cache_was_used = true; - } - else - { - /// In case of reading only offset use the correct serialization for reading of the prefix - auto serialization = getSerializationInPart(name_type_in_storage); - temp_column = name_type_in_storage.type->createColumn(*serialization); - - if (column_for_offsets) - { - auto serialization_for_prefix = getSerializationInPart(*column_for_offsets); - - deserialize_settings.getter = buffer_getter_for_prefix; - serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); - } - - deserialize_settings.getter = buffer_getter; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, state); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, state, nullptr); - - if (!column_for_offsets) - columns_cache_for_subcolumns[name_type_in_storage.name] = temp_column; - - auto subcolumn = name_type_in_storage.type->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. if (column->empty()) @@ -381,185 +151,98 @@ void MergeTreeReaderCompact::readData( else column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); } - } - else - { - /// In case of reading only offsets use the correct serialization for reading the prefix - auto serialization = getSerializationInPart(name_and_type); - - if (column_for_offsets) + else { - auto serialization_for_prefix = getSerializationInPart(*column_for_offsets); - - deserialize_settings.getter = buffer_getter_for_prefix; - serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); + auto serialization = getSerializationInPart(name_and_type); + serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); } - deserialize_settings.getter = buffer_getter; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, state); - serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state, nullptr); - } - - /// The buffer is left in inconsistent state after reading single offsets or using columns cache during subcolumns reading. - if (name_level_for_offsets.has_value() || columns_cache_was_used) - last_read_granule.reset(); - else - last_read_granule.emplace(from_mark, column_position); -} - -void MergeTreeReaderCompact::prefetchBeginOfRange(Priority priority) -try -{ - if (!initialized) - { - initialize(); - initialized = true; - } - - adjustUpperBound(all_mark_ranges.back().end); - seekToMark(all_mark_ranges.front().begin, 0); - data_buffer->prefetch(priority); -} -catch (...) -{ - if (!isRetryableException(std::current_exception())) - data_part_info_for_read->reportBroken(); - throw; -} - -void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) -{ - MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); - try - { - compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + size_t read_rows_in_column = column->size() - column_size_before_reading; + if (read_rows_in_column != rows_to_read) + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read all data in MergeTreeReaderCompact. Rows read: {}. Rows expected: {}.", + read_rows_in_column, rows_to_read); } catch (Exception & e) { - /// Better diagnostics. - if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) - e.addMessage("(while seeking to mark (" + toString(row_index) + ", " + toString(column_index) + ")"); - + e.addMessage("(while reading column " + name_and_type.name + ")"); throw; } } -void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) + +void MergeTreeReaderCompact::readPrefix( + const NameAndTypePair & name_and_type, + const InputStreamGetter & buffer_getter, + const InputStreamGetter & buffer_getter_for_prefix, + const ColumnNameLevel & name_level_for_offsets) { - size_t right_offset = 0; - if (last_mark < data_part_info_for_read->getMarksCount()) /// Otherwise read until the end of file - right_offset = marks_loader.getMark(last_mark).offset_in_compressed_file; - - if (right_offset == 0) + try { - /// If already reading till the end of file. - if (last_right_offset && *last_right_offset == 0) - return; + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; - last_right_offset = 0; // Zero value means the end of file. - data_buffer->setReadUntilEnd(); + if (name_level_for_offsets.has_value()) + { + const auto & part_columns = data_part_info_for_read->getColumnsDescription(); + auto column_for_offsets = part_columns.getPhysical(name_level_for_offsets->first); + + auto serialization_for_prefix = getSerializationInPart(column_for_offsets); + deserialize_settings.getter = buffer_getter_for_prefix; + ISerialization::DeserializeBinaryBulkStatePtr state_for_prefix; + + serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix); + } + + SerializationPtr serialization; + if (name_and_type.isSubcolumn()) + serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); + else + serialization = getSerializationInPart(name_and_type); + + + deserialize_settings.getter = buffer_getter; + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.name]); } - else + catch (Exception & e) { - if (last_right_offset && right_offset <= last_right_offset.value()) - return; - - last_right_offset = right_offset; - data_buffer->setReadUntilPosition(right_offset); + e.addMessage("(while reading column " + name_and_type.name + ")"); + throw; } } -bool MergeTreeReaderCompact::isContinuousReading(size_t mark, size_t column_position) +void MergeTreeReaderCompact::createColumnsForReading(Columns & res_columns) const { - if (!last_read_granule) + for (size_t i = 0; i < columns_to_read.size(); ++i) + { + if (column_positions[i] && res_columns[i] == nullptr) + res_columns[i] = columns_to_read[i].type->createColumn(*serializations[i]); + } +} + +bool MergeTreeReaderCompact::needSkipStream(size_t column_pos, const ISerialization::SubstreamPath & substream) const +{ + /// Offset stream can be read only from columns of current level or + /// below (since it is OK to read all parent streams from the + /// alternative). + /// + /// Consider the following columns in nested "root": + /// - root.array Array(UInt8) - exists + /// - root.nested_array Array(Array(UInt8)) - does not exists (only_offsets_level=1) + /// + /// For root.nested_array it will try to read multiple streams: + /// - offsets (substream_path = {ArraySizes}) + /// OK + /// - root.nested_array elements (substream_path = {ArrayElements, ArraySizes}) + /// NOT OK - cannot use root.array offsets stream for this + /// + /// Here only_offsets_level is the level of the alternative stream, + /// and substream_path.size() is the level of the current stream. + + if (!columns_for_offsets[column_pos]) return false; - const auto & [last_mark, last_column] = *last_read_granule; - return (mark == last_mark && column_position == last_column + 1) - || (mark == last_mark + 1 && column_position == 0 && last_column == data_part_info_for_read->getColumns().size() - 1); -} -namespace -{ - -/// A simple class that helps to iterate over 2-dim marks of compact parts. -class MarksCounter -{ -public: - MarksCounter(size_t rows_num_, size_t columns_num_) - : rows_num(rows_num_), columns_num(columns_num_) {} - - struct Iterator - { - size_t row; - size_t column; - MarksCounter * counter; - - Iterator(size_t row_, size_t column_, MarksCounter * counter_) - : row(row_), column(column_), counter(counter_) {} - - Iterator operator++() - { - if (column + 1 == counter->columns_num) - { - ++row; - column = 0; - } - else - { - ++column; - } - - return *this; - } - - bool operator==(const Iterator & other) const { return row == other.row && column == other.column; } - bool operator!=(const Iterator & other) const { return !(*this == other); } - }; - - Iterator get(size_t row, size_t column) { return Iterator(row, column, this); } - Iterator end() { return get(rows_num, 0); } - -private: - size_t rows_num; - size_t columns_num; -}; - -} - -size_t MergeTreeReaderCompact::getReadBufferSize( - const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, - MergeTreeMarksLoader & marks_loader, - const ColumnPositions & column_positions, - const MarkRanges & mark_ranges) -{ - size_t buffer_size = 0; - size_t columns_num = column_positions.size(); - size_t file_size = data_part_info_for_reader.getFileSizeOrZero(MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION); - - MarksCounter counter(data_part_info_for_reader.getMarksCount(), data_part_info_for_reader.getColumns().size()); - - for (const auto & mark_range : mark_ranges) - { - for (size_t mark = mark_range.begin; mark < mark_range.end; ++mark) - { - for (size_t i = 0; i < columns_num; ++i) - { - if (!column_positions[i]) - continue; - - auto it = counter.get(mark, *column_positions[i]); - size_t cur_offset = marks_loader.getMark(it.row, it.column).offset_in_compressed_file; - - while (it != counter.end() && cur_offset == marks_loader.getMark(it.row, it.column).offset_in_compressed_file) - ++it; - - size_t next_offset = (it == counter.end() ? file_size : marks_loader.getMark(it.row, it.column).offset_in_compressed_file); - buffer_size = std::max(buffer_size, next_offset - cur_offset); - } - } - } - - return buffer_size; + bool is_offsets = !substream.empty() && substream.back().type == ISerialization::Substream::ArraySizes; + return !is_offsets || columns_for_offsets[column_pos]->second < ISerialization::getArrayLevel(substream); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 769e6a08be4..9c66b4691a3 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -3,7 +3,7 @@ #include #include #include - +#include namespace DB { @@ -14,7 +14,7 @@ using DataPartCompactPtr = std::shared_ptr; class IMergeTreeDataPart; using DataPartPtr = std::shared_ptr; -/// Reader for compact parts +/// Base class of readers for compact parts. class MergeTreeReaderCompact : public IMergeTreeReader { public: @@ -27,31 +27,37 @@ public: MarkCache * mark_cache_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_, - ThreadPool * load_marks_threadpool_, - ValueSizeMap avg_value_size_hints_ = {}, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = {}, - clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + ValueSizeMap avg_value_size_hints_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_); - /// Return the number of rows has been read or zero if there is no columns to read. - /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - size_t readRows(size_t from_mark, size_t current_task_last_mark, - bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; + bool canReadIncompleteGranules() const override final { return false; } - bool canReadIncompleteGranules() const override { return false; } - - void prefetchBeginOfRange(Priority priority) override; - -private: - bool isContinuousReading(size_t mark, size_t column_position); +protected: void fillColumnPositions(); - void initialize(); - ReadBuffer * data_buffer; - CompressedReadBufferBase * compressed_data_buffer; - std::unique_ptr cached_buffer; - std::unique_ptr non_cached_buffer; + using InputStreamGetter = ISerialization::InputStreamGetter; - MergeTreeMarksLoader marks_loader; + void readData( + const NameAndTypePair & name_and_type, + ColumnPtr & column, + size_t rows_to_read, + const InputStreamGetter & getter); + + void readPrefix( + const NameAndTypePair & name_and_type, + const InputStreamGetter & buffer_getter, + const InputStreamGetter & buffer_getter_for_prefix, + const ColumnNameLevel & name_level_for_offsets); + + void createColumnsForReading(Columns & res_columns) const; + bool needSkipStream(size_t column_pos, const ISerialization::SubstreamPath & substream) const; + + const MergeTreeMarksLoaderPtr marks_loader; + MergeTreeMarksGetterPtr marks_getter; + + ReadBufferFromFileBase::ProfileCallback profile_callback; + clockid_t clock_type; /// Storage columns with collected separate arrays of Nested to columns of Nested type. /// They maybe be needed for finding offsets of missed Nested columns in parts. @@ -67,32 +73,9 @@ private: /// Element of the vector is the level of the alternative stream. std::vector columns_for_offsets; - /// For asynchronous reading from remote fs. Same meaning as in MergeTreeReaderStream. - std::optional last_right_offset; - + /// Mark to read in next 'readRows' call in case, + /// when 'continue_reading' is true. size_t next_mark = 0; - std::optional> last_read_granule; - - void seekToMark(size_t row_index, size_t column_index); - - void readData(const NameAndTypePair & name_and_type, ColumnPtr & column, size_t from_mark, - size_t current_task_last_mark, size_t column_position, - size_t rows_to_read, ColumnNameLevel name_level_for_offsets, std::unordered_map & columns_cache_for_subcolumns); - - /// Returns maximal value of granule size in compressed file from @mark_ranges. - /// This value is used as size of read buffer. - static size_t getReadBufferSize( - const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, - MergeTreeMarksLoader & marks_loader, - const ColumnPositions & column_positions, - const MarkRanges & mark_ranges); - - /// For asynchronous reading from remote fs. - void adjustUpperBound(size_t last_mark); - - ReadBufferFromFileBase::ProfileCallback profile_callback; - clockid_t clock_type; - bool initialized = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp new file mode 100644 index 00000000000..2b2cf493bb5 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +size_t MergeTreeReaderCompactSingleBuffer::readRows( + size_t from_mark, size_t current_task_last_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) +try +{ + init(); + + if (continue_reading) + from_mark = next_mark; + + size_t read_rows = 0; + size_t num_columns = columns_to_read.size(); + + checkNumberOfColumns(num_columns); + createColumnsForReading(res_columns); + + while (read_rows < max_rows_to_read) + { + size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); + + for (size_t pos = 0; pos < num_columns; ++pos) + { + if (!res_columns[pos]) + continue; + + auto & column = res_columns[pos]; + + stream->adjustRightMark(current_task_last_mark); /// Must go before seek. + stream->seekToMarkAndColumn(from_mark, *column_positions[pos]); + + auto buffer_getter = [&](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * + { + if (needSkipStream(pos, substream_path)) + return nullptr; + + return stream->getDataBuffer(); + }; + + /// If we read only offsets we have to read prefix anyway + /// to preserve correctness of serialization. + auto buffer_getter_for_prefix = [&](const auto &) -> ReadBuffer * + { + return stream->getDataBuffer(); + }; + + readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter); + } + + ++from_mark; + read_rows += rows_to_read; + } + + next_mark = from_mark; + return read_rows; +} +catch (...) +{ + if (!isRetryableException(std::current_exception())) + data_part_info_for_read->reportBroken(); + + /// Better diagnostics. + try + { + rethrow_exception(std::current_exception()); + } + catch (Exception & e) + { + e.addMessage(getMessageForDiagnosticOfBrokenPart(from_mark, max_rows_to_read)); + } + + throw; +} + +void MergeTreeReaderCompactSingleBuffer::init() +try +{ + if (initialized) + return; + + auto stream_settings = settings; + stream_settings.allow_different_codecs = true; + + stream = std::make_unique( + data_part_info_for_read->getDataPartStorage(), MergeTreeDataPartCompact::DATA_FILE_NAME, + MergeTreeDataPartCompact::DATA_FILE_EXTENSION, data_part_info_for_read->getMarksCount(), + all_mark_ranges, stream_settings,uncompressed_cache, + data_part_info_for_read->getFileSizeOrZero(MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION), + marks_loader, profile_callback, clock_type); + + initialized = true; +} +catch (...) +{ + if (!isRetryableException(std::current_exception())) + data_part_info_for_read->reportBroken(); + throw; +} + +} diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h new file mode 100644 index 00000000000..f358120e717 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.h @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +namespace DB +{ + +/// Reader for compact parts, that uses one buffer for +/// all column and doesn't support parallel prefetch of columns. +/// It's suitable for compact parts with small size of stripe. +class MergeTreeReaderCompactSingleBuffer : public MergeTreeReaderCompact +{ +public: + template + explicit MergeTreeReaderCompactSingleBuffer(Args &&... args) + : MergeTreeReaderCompact{std::forward(args)...} + { + fillColumnPositions(); + } + + /// Returns the number of rows has been read or zero if there is no columns to read. + /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark + size_t readRows(size_t from_mark, size_t current_task_last_mark, + bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; + +private: + void init(); + + bool initialized = false; + std::unique_ptr stream; +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 8e460630d5e..40a16176c69 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -16,43 +16,28 @@ namespace ErrorCodes } MergeTreeReaderStream::MergeTreeReaderStream( - MergeTreeDataPartInfoForReaderPtr data_part_reader_, - const String & path_prefix_, - const String & data_file_extension_, - size_t marks_count_, - const MarkRanges & all_mark_ranges_, - const MergeTreeReaderSettings & settings_, - MarkCache * mark_cache_, - UncompressedCache * uncompressed_cache_, - size_t file_size_, - const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, - clockid_t clock_type_, - bool is_low_cardinality_dictionary_, - ThreadPool * load_marks_cache_threadpool_) - : settings(settings_) - , profile_callback(profile_callback_) + DataPartStoragePtr data_part_storage_, + const String & path_prefix_, + const String & data_file_extension_, + size_t marks_count_, + const MarkRanges & all_mark_ranges_, + const MergeTreeReaderSettings & settings_, + UncompressedCache * uncompressed_cache_, + size_t file_size_, + MergeTreeMarksLoaderPtr marks_loader_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_) + : profile_callback(profile_callback_) , clock_type(clock_type_) , all_mark_ranges(all_mark_ranges_) - , file_size(file_size_) - , uncompressed_cache(uncompressed_cache_) - , data_part_storage(data_part_reader_->getDataPartStorage()) + , data_part_storage(std::move(data_part_storage_)) , path_prefix(path_prefix_) , data_file_extension(data_file_extension_) - , is_low_cardinality_dictionary(is_low_cardinality_dictionary_) + , uncompressed_cache(uncompressed_cache_) + , settings(settings_) , marks_count(marks_count_) - , mark_cache(mark_cache_) - , save_marks_in_cache(settings.save_marks_in_cache) - , index_granularity_info(index_granularity_info_) - , marks_loader( - data_part_reader_, - mark_cache, - index_granularity_info->getMarksFilePath(path_prefix), - marks_count, - *index_granularity_info, - save_marks_in_cache, - settings.read_settings, - load_marks_cache_threadpool_) + , file_size(file_size_) + , marks_loader(std::move(marks_loader_)) { } @@ -60,21 +45,12 @@ void MergeTreeReaderStream::init() { if (initialized) return; + initialized = true; + marks_getter = marks_loader->loadMarks(); + /// Compute the size of the buffer. - size_t max_mark_range_bytes = 0; - size_t sum_mark_range_bytes = 0; - - for (const auto & mark_range : all_mark_ranges) - { - size_t left_mark = mark_range.begin; - size_t right_mark = mark_range.end; - size_t left_offset = left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0; - auto mark_range_bytes = getRightOffset(right_mark) - left_offset; - - max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); - sum_mark_range_bytes += mark_range_bytes; - } + auto [max_mark_range_bytes, sum_mark_range_bytes] = estimateMarkRangeBytes(all_mark_ranges); std::optional estimated_sum_mark_range_bytes; if (sum_mark_range_bytes) @@ -83,7 +59,7 @@ void MergeTreeReaderStream::init() /// Avoid empty buffer. May happen while reading dictionary for DataTypeLowCardinality. /// For example: part has single dictionary and all marks point to the same position. ReadSettings read_settings = settings.read_settings; - if (max_mark_range_bytes != 0) + if (settings.adjust_read_buffer_size && max_mark_range_bytes != 0) read_settings = read_settings.adjustBufferSize(max_mark_range_bytes); //// Empty buffer does not makes progress. @@ -102,7 +78,8 @@ void MergeTreeReaderStream::init() read_settings, estimated_sum_mark_range_bytes, std::nullopt); }, - uncompressed_cache); + uncompressed_cache, + settings.allow_different_codecs); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -121,7 +98,7 @@ void MergeTreeReaderStream::init() path_prefix + data_file_extension, read_settings, estimated_sum_mark_range_bytes, - std::nullopt)); + std::nullopt), settings.allow_different_codecs); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -135,99 +112,10 @@ void MergeTreeReaderStream::init() } } - -size_t MergeTreeReaderStream::getRightOffset(size_t right_mark) -{ - /// NOTE: if we are reading the whole file, then right_mark == marks_count - /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. - - /// Special case, can happen in Collapsing/Replacing engines - if (marks_count == 0) - return 0; - - assert(right_mark <= marks_count); - - if (0 < right_mark && right_mark < marks_count) - { - /// Find the right border of the last mark we need to read. - /// To do that let's find the upper bound of the offset of the last - /// included mark. - - if (is_low_cardinality_dictionary) - { - - /// In LowCardinality dictionary several consecutive marks can point to the same offset. - /// - /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) - /// one granule may require reading of two dictionaries which starts from different marks. - /// The only correct way is to take offset from at least next different granule from the right one. - /// So, that's why we have to read one extra granule to the right, - /// while reading dictionary of LowCardinality. - /// - /// Example: - /// Mark 0, points to [0, 8] - /// Mark 1, points to [0, 8] - /// Mark 2, points to [0, 8] - /// Mark 3, points to [0, 8] - /// Mark 4, points to [42336, 2255] - /// Mark 5, points to [42336, 2255] <--- for example need to read until 5 - /// Mark 6, points to [42336, 2255] <--- not suitable, because have same offset - /// Mark 7, points to [84995, 7738] <--- next different mark - /// Mark 8, points to [84995, 7738] - /// Mark 9, points to [126531, 8637] <--- what we are looking for - - auto indices = collections::range(right_mark, marks_count); - auto next_different_mark = [&](auto lhs, auto rhs) - { - return marks_loader.getMark(lhs).asTuple() < marks_loader.getMark(rhs).asTuple(); - }; - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); - - if (it == indices.end()) - return file_size; - - right_mark = *it; - } - - /// This is a good scenario. The compressed block is finished within the right mark, - /// and previous mark was different. - if (marks_loader.getMark(right_mark).offset_in_decompressed_block == 0 - && marks_loader.getMark(right_mark) != marks_loader.getMark(right_mark - 1)) - return marks_loader.getMark(right_mark).offset_in_compressed_file; - - /// If right_mark has non-zero offset in decompressed block, we have to - /// read its compressed block in a whole, because it may consist of data from previous granule. - /// - /// For example: - /// Mark 6, points to [42336, 2255] - /// Mark 7, points to [84995, 7738] <--- right_mark - /// Mark 8, points to [84995, 7738] - /// Mark 9, points to [126531, 8637] <--- what we are looking for - /// - /// Since mark 7 starts from offset in decompressed block 7738, - /// it has some data from mark 6 and we have to read - /// compressed block [84995; 126531 in a whole. - - auto indices = collections::range(right_mark, marks_count); - auto next_different_compressed_offset = [&](auto lhs, auto rhs) - { - return marks_loader.getMark(lhs).offset_in_compressed_file < marks_loader.getMark(rhs).offset_in_compressed_file; - }; - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_compressed_offset)); - - if (it != indices.end()) - return marks_loader.getMark(*it).offset_in_compressed_file; - } - else if (right_mark == 0) - return marks_loader.getMark(right_mark).offset_in_compressed_file; - - return file_size; -} - -void MergeTreeReaderStream::seekToMark(size_t index) +void MergeTreeReaderStream::seekToMarkAndColumn(size_t row_index, size_t column_position) { init(); - MarkInCompressedFile mark = marks_loader.getMark(index); + const auto & mark = marks_getter->getMark(row_index, column_position); try { @@ -237,7 +125,7 @@ void MergeTreeReaderStream::seekToMark(size_t index) { /// Better diagnostics. if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) - e.addMessage("(while seeking to mark " + toString(index) + e.addMessage("(while seeking to mark " + toString(row_index) + " of column " + path_prefix + "; offsets are: " + toString(mark.offset_in_compressed_file) + " " + toString(mark.offset_in_decompressed_block) + ")"); @@ -274,6 +162,7 @@ void MergeTreeReaderStream::adjustRightMark(size_t right_mark) */ init(); auto right_offset = getRightOffset(right_mark); + if (!right_offset) { if (last_right_offset && *last_right_offset == 0) @@ -304,4 +193,276 @@ CompressedReadBufferBase * MergeTreeReaderStream::getCompressedDataBuffer() return compressed_data_buffer; } +size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) const +{ + /// NOTE: if we are reading the whole file, then right_mark == marks_count + /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. + + /// Special case, can happen in Collapsing/Replacing engines + if (marks_count == 0) + return 0; + + assert(right_mark <= marks_count); + + if (right_mark == 0) + return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; + + if (right_mark == marks_count) + return file_size; + + /// Find the right border of the last mark we need to read. + /// To do that let's find the upper bound of the offset of the last + /// included mark. + + if (settings.is_low_cardinality_dictionary) + { + /// In LowCardinality dictionary several consecutive marks can point to the same offset. + /// + /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) + /// one granule may require reading of two dictionaries which starts from different marks. + /// The only correct way is to take offset from at least next different granule from the right one. + /// So, that's why we have to read one extra granule to the right, + /// while reading dictionary of LowCardinality. + /// + /// Example: + /// Mark 0, points to [0, 8] + /// Mark 1, points to [0, 8] + /// Mark 2, points to [0, 8] + /// Mark 3, points to [0, 8] + /// Mark 4, points to [42336, 2255] + /// Mark 5, points to [42336, 2255] <--- for example need to read until 5 + /// Mark 6, points to [42336, 2255] <--- not suitable, because have same offset + /// Mark 7, points to [84995, 7738] <--- next different mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for + + auto indices = collections::range(right_mark, marks_count); + auto next_different_mark = [&](auto lhs, auto rhs) + { + return marks_getter->getMark(lhs, 0).asTuple() < marks_getter->getMark(rhs, 0).asTuple(); + }; + + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); + if (it == indices.end()) + return file_size; + + right_mark = *it; + } + + /// This is a good scenario. The compressed block is finished within the right mark, + /// and previous mark was different. + if (marks_getter->getMark(right_mark, 0).offset_in_decompressed_block == 0 + && marks_getter->getMark(right_mark, 0) != marks_getter->getMark(right_mark - 1, 0)) + return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; + + /// If right_mark has non-zero offset in decompressed block, we have to + /// read its compressed block in a whole, because it may consist of data from previous granule. + /// + /// For example: + /// Mark 6, points to [42336, 2255] + /// Mark 7, points to [84995, 7738] <--- right_mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for + /// + /// Since mark 7 starts from offset in decompressed block 7738, + /// it has some data from mark 6 and we have to read + /// compressed block [84995; 126531 in a whole. + + auto indices = collections::range(right_mark, marks_count); + auto next_different_compressed_offset = [&](auto lhs, auto rhs) + { + return marks_getter->getMark(lhs, 0).offset_in_compressed_file < marks_getter->getMark(rhs, 0).offset_in_compressed_file; + }; + + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_compressed_offset)); + if (it != indices.end()) + return marks_getter->getMark(*it, 0).offset_in_compressed_file; + + return file_size; +} + +std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + assert(marks_getter != nullptr); + + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (const auto & mark_range : mark_ranges) + { + size_t left_mark = mark_range.begin; + size_t right_mark = mark_range.end; + size_t left_offset = left_mark < marks_count ? marks_getter->getMark(left_mark, 0).offset_in_compressed_file : 0; + auto mark_range_bytes = getRightOffset(right_mark) - left_offset; + + max_range_bytes = std::max(max_range_bytes, mark_range_bytes); + sum_range_bytes += mark_range_bytes; + } + + return {max_range_bytes, sum_range_bytes}; +} + +size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const +{ + /// NOTE: if we are reading the whole file, then right_mark == marks_count + /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. + + /// Special case, can happen in Collapsing/Replacing engines + if (marks_count == 0) + return 0; + + assert(right_mark_non_included <= marks_count); + + if (right_mark_non_included == 0) + return marks_getter->getMark(right_mark_non_included, column_position).offset_in_compressed_file; + + size_t right_mark_included = right_mark_non_included - 1; + if (right_mark_non_included != marks_count + && marks_getter->getMark(right_mark_non_included, column_position).offset_in_decompressed_block != 0) + ++right_mark_included; + + /// The right bound for case, where there is no smaller suitable mark + /// is the start of the next stripe (in which the next column is written) + /// because each stripe always start from a new compressed block. + const auto & right_mark_in_file = marks_getter->getMark(right_mark_included, column_position); + auto next_stripe_right_mark_in_file = getStartOfNextStripeMark(right_mark_included, column_position); + + /// Try to find suitable right mark in current stripe. + for (size_t mark = right_mark_included + 1; mark < marks_count; ++mark) + { + const auto & current_mark = marks_getter->getMark(mark, column_position); + /// We found first mark that starts from the new compressed block. + if (current_mark.offset_in_compressed_file > right_mark_in_file.offset_in_compressed_file) + { + /// If it is in current stripe return it to reduce amount of read data. + if (current_mark < next_stripe_right_mark_in_file) + return current_mark.offset_in_compressed_file; + + /// Otherwise return start of new stripe as an upper bound. + break; + } + } + + return next_stripe_right_mark_in_file.offset_in_compressed_file; +} + +std::pair +MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const +{ + assert(marks_getter != nullptr); + + /// As a maximal range we return the maximal size of a whole stripe. + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (const auto & mark_range : mark_ranges) + { + auto start_of_stripe_mark = marks_getter->getMark(mark_range.begin, column_position); + auto start_of_next_stripe_mark = getStartOfNextStripeMark(mark_range.begin, column_position); + + for (size_t mark = mark_range.begin; mark < mark_range.end; ++mark) + { + const auto & current_mark = marks_getter->getMark(mark, column_position); + + /// We found a start of new stripe, now update values. + if (current_mark > start_of_next_stripe_mark) + { + auto current_range_bytes = getRightOffsetOneColumn(mark, column_position) - start_of_stripe_mark.offset_in_compressed_file; + + max_range_bytes = std::max(max_range_bytes, current_range_bytes); + sum_range_bytes += current_range_bytes; + + start_of_stripe_mark = current_mark; + start_of_next_stripe_mark = getStartOfNextStripeMark(mark, column_position); + } + } + + auto current_range_bytes = getRightOffsetOneColumn(mark_range.end, column_position) - start_of_stripe_mark.offset_in_compressed_file; + + max_range_bytes = std::max(max_range_bytes, current_range_bytes); + sum_range_bytes += current_range_bytes; + } + + return {max_range_bytes, sum_range_bytes}; +} + +MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) const +{ + const auto & current_mark = marks_getter->getMark(row_index, column_position); + + if (marks_getter->getNumColumns() == 1) + return MarkInCompressedFile{file_size, 0}; + + if (column_position + 1 == marks_getter->getNumColumns()) + { + /** + * In case of the last column (c3), we have the following picture: + * c1 c2 c3 + * x x x + * (row_index, 0) -> o x o <- (row_index, column_position) + * x x x + * ------- <- start of new stripe + * what we are -> o x x + * looking for x x x + * x x x + * ------- + * So, we need to iterate forward. + */ + size_t mark_index = row_index + 1; + while (mark_index < marks_count && marks_getter->getMark(mark_index, 0) <= current_mark) + ++mark_index; + + return mark_index == marks_count + ? MarkInCompressedFile{file_size, 0} + : marks_getter->getMark(mark_index, 0); + } + + /** + * Otherwise, we have the following picture: + * c1 c2 c3 + * x x o <- what we are looking for + * (row, column) --> o o <- (row, column + 1) + * x x x + * ------- <- start of new stripe + * So, we need to iterate backward. + */ + + ssize_t mark_index = row_index; + while (mark_index >= 0 && marks_getter->getMark(mark_index, column_position + 1) >= current_mark) + --mark_index; + + return marks_getter->getMark(mark_index + 1, column_position + 1); +} + +size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +{ + return getRightOffsetOneColumn(right_mark_non_included, column_position); +} + +std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + return estimateMarkRangeBytesOneColumn(mark_ranges, column_position); +} + +size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +{ + return getRightOffsetOneColumn(right_mark_non_included, marks_loader->getNumColumns() - 1); +} + +std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +{ + size_t max_range_bytes = 0; + size_t sum_range_bytes = 0; + + for (size_t i = 0; i < marks_getter->getNumColumns(); ++i) + { + auto [current_max, current_sum] = estimateMarkRangeBytesOneColumn(mark_ranges, i); + + max_range_bytes = std::max(max_range_bytes, current_max); + sum_range_bytes += current_sum; + } + + return {max_range_bytes, sum_range_bytes}; +} + } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 49ce3103434..f3ca6953ceb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -14,27 +14,31 @@ namespace DB { -/// Class for reading a single column (or index). +/// Basic and the most low-level class +/// for reading single columns or indexes. class MergeTreeReaderStream { public: MergeTreeReaderStream( - MergeTreeDataPartInfoForReaderPtr data_part_reader_, + DataPartStoragePtr data_part_storage_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, - const MarkRanges & all_mark_ranges, + const MarkRanges & all_mark_ranges_, const MergeTreeReaderSettings & settings_, - MarkCache * mark_cache, - UncompressedCache * uncompressed_cache, + UncompressedCache * uncompressed_cache_, size_t file_size_, - const MergeTreeIndexGranularityInfo * index_granularity_info_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback, - clockid_t clock_type, - bool is_low_cardinality_dictionary_, - ThreadPool * load_marks_cache_threadpool_); + MergeTreeMarksLoaderPtr marks_loader_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_); - void seekToMark(size_t index); + virtual ~MergeTreeReaderStream() = default; + + /// Seeks to start of @row_index mark. Column position is implementation defined. + virtual void seekToMark(size_t row_index) = 0; + + /// Seeks to exact mark in file. + void seekToMarkAndColumn(size_t row_index, size_t column_position); void seekToStart(); @@ -48,39 +52,111 @@ public: CompressedReadBufferBase * getCompressedDataBuffer(); private: - void init(); - size_t getRightOffset(size_t right_mark); + /// Returns offset in file up to which it's needed to read file to read all rows up to @right_mark mark. + virtual size_t getRightOffset(size_t right_mark) const = 0; + + /// Returns estimated max amount of bytes to read among mark ranges (which is used as size for read buffer) + /// and total amount of bytes to read in all mark ranges. + virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const = 0; - const MergeTreeReaderSettings settings; const ReadBufferFromFileBase::ProfileCallback profile_callback; - clockid_t clock_type; + const clockid_t clock_type; const MarkRanges all_mark_ranges; - size_t file_size; - UncompressedCache * uncompressed_cache; - DataPartStoragePtr data_part_storage; - std::string path_prefix; - std::string data_file_extension; - - bool is_low_cardinality_dictionary = false; - - size_t marks_count; + const DataPartStoragePtr data_part_storage; + const std::string path_prefix; + const std::string data_file_extension; + UncompressedCache * const uncompressed_cache; ReadBuffer * data_buffer; CompressedReadBufferBase * compressed_data_buffer; - MarkCache * mark_cache; - bool save_marks_in_cache; + bool initialized = false; - std::optional last_right_offset; - const MergeTreeIndexGranularityInfo * index_granularity_info; - std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; - MergeTreeMarksLoader marks_loader; +protected: + void init(); + + const MergeTreeReaderSettings settings; + const size_t marks_count; + const size_t file_size; + + const MergeTreeMarksLoaderPtr marks_loader; + MergeTreeMarksGetterPtr marks_getter; +}; + +/// Class for reading a single column (or index) from file +/// that contains a single column (for wide parts). +class MergeTreeReaderStreamSingleColumn : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamSingleColumn(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } +}; + +/// Base class for reading from file that contains multiple columns. +/// It is used to read from compact parts. +/// See more details about data layout in MergeTreeDataPartCompact.h. +class MergeTreeReaderStreamMultipleColumns : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamMultipleColumns(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + +protected: + size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const; + std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const; + MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position) const; +}; + +/// Class for reading a single column from file that contains multiple columns +/// (for parallel reading from compact parts with large stripes). +class MergeTreeReaderStreamOneOfMultipleColumns : public MergeTreeReaderStreamMultipleColumns +{ +public: + template + explicit MergeTreeReaderStreamOneOfMultipleColumns(size_t column_position_, Args &&... args) + : MergeTreeReaderStreamMultipleColumns{std::forward(args)...} + , column_position(column_position_) + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, column_position); } + +private: + const size_t column_position; +}; + +/// Class for reading multiple columns from file that contains multiple columns +/// (for reading from compact parts with small stripes). +class MergeTreeReaderStreamAllOfMultipleColumns : public MergeTreeReaderStreamMultipleColumns +{ +public: + template + explicit MergeTreeReaderStreamAllOfMultipleColumns(Args &&... args) + : MergeTreeReaderStreamMultipleColumns{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) const override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; } diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index d34a58a25b0..394a22835f1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -225,18 +225,29 @@ void MergeTreeReaderWide::addStreams( return; } - has_any_stream = true; - bool is_lc_dict = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; - auto context = data_part_info_for_read->getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; - streams.emplace(*stream_name, std::make_unique( - data_part_info_for_read, *stream_name, DATA_FILE_EXTENSION, - data_part_info_for_read->getMarksCount(), all_mark_ranges, settings, mark_cache, + auto marks_loader = std::make_shared( + data_part_info_for_read, + mark_cache, + data_part_info_for_read->getIndexGranularityInfo().getMarksFilePath(*stream_name), + data_part_info_for_read->getMarksCount(), + data_part_info_for_read->getIndexGranularityInfo(), + settings.save_marks_in_cache, + settings.read_settings, + load_marks_threadpool, + /*num_columns_in_mark=*/ 1); + + has_any_stream = true; + auto stream_settings = settings; + stream_settings.is_low_cardinality_dictionary = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; + + streams.emplace(*stream_name, std::make_unique( + data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, + data_part_info_for_read->getMarksCount(), all_mark_ranges, stream_settings, uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), - &data_part_info_for_read->getIndexGranularityInfo(), - profile_callback, clock_type, is_lc_dict, load_marks_threadpool)); + std::move(marks_loader), profile_callback, clock_type)); }; serialization->enumerateStreams(callback); diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp index 5f89849e920..329275f4605 100644 --- a/src/Storages/StorageMergeTreeIndex.cpp +++ b/src/Storages/StorageMergeTreeIndex.cpp @@ -184,9 +184,11 @@ private: auto & compressed_data = compressed->getData(); auto & uncompressed_data = uncompressed->getData(); + auto marks_getter = marks_loader->loadMarks(); + for (size_t i = 0; i < num_rows; ++i) { - auto mark = marks_loader->getMark(i, col_idx); + auto mark = marks_getter->getMark(i, col_idx); compressed_data[i] = mark.offset_in_compressed_file; uncompressed_data[i] = mark.offset_in_decompressed_block; From 265d91747fceb596d76ea7422f6dfa6995b61870 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:03:12 +0000 Subject: [PATCH 137/332] Remove log debug --- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 62 +++++-------------- src/Storages/System/StorageSystemNumbers.cpp | 2 - src/Storages/System/attachSystemTables.cpp | 1 + .../TableFunctionGenerateSeries.cpp | 23 ++++--- src/TableFunctions/TableFunctionNumbers.cpp | 11 ++-- 5 files changed, 35 insertions(+), 64 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 52b5eeb2ca3..503dd534d89 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -29,6 +29,15 @@ extern const int TOO_MANY_ROWS; namespace { +template +inline void iota_with_step_optimized(T * begin, size_t count, T first_value, T step) +{ + if (step == 1) + iota(begin, count, first_value); + else + iota_with_step(begin, count, first_value, step); +} + class NumbersSource : public ISource { public: @@ -66,14 +75,8 @@ protected: UInt64 * pos = vec.data(); /// This also accelerates the code. UInt64 * current_end = &vec[real_block_size]; - if (step == 1) - { - iota(pos, static_cast(current_end - pos), curr); - } - else - { - iota_with_step(pos, static_cast(current_end - pos), curr, step); - } + + iota_with_step_optimized(pos, static_cast(current_end - pos), curr, step); next += chunk_step; @@ -101,8 +104,6 @@ using RangesWithStep = std::vector; std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) { - // LOG_DEBUG(&Poco::Logger::get("Stepped from range"), - // "stepped from range"); if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; UInt64 begin = (r.left.get() / step) * step; @@ -155,7 +156,7 @@ public: using RangesStatePtr = std::shared_ptr; - [[maybe_unused]] NumbersRangedSource( + NumbersRangedSource( const RangesWithStep & ranges_, RangesStatePtr & ranges_state_, UInt64 base_block_size_, @@ -167,17 +168,6 @@ public: , base_block_size(base_block_size_) , step(step_) { - // for (const auto& range_with_step : ranges_) { - // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), - // // "Ranges: {} {} {} {} {}", - // // range_with_step.range.left.get(), - // // range_with_step.range.right.get(), - // // range_with_step.range.left_included, - // // range_with_step.range.right_included, - // // range_with_step.step); - // // LOG_DEBUG(&Poco::Logger::get("Ranges With Step"), - // // "Step: {}", step); - // } } String getName() const override { return "NumbersRange"; } @@ -242,8 +232,6 @@ protected: RangesPos start, end; auto block_size = findRanges(start, end, base_block_size); - // LOG_DEBUG(&Poco::Logger::get("Found range"), "Evth: {} {} {} {} {} {}", start.offset_in_ranges, static_cast(start.offset_in_range), end.offset_in_ranges, static_cast(end.offset_in_range), base_block_size, block_size); - if (!block_size) return {}; @@ -259,11 +247,6 @@ protected: while (block_size - provided != 0) { UInt64 need = block_size - provided; - // LOG_DEBUG(&Poco::Logger::get("Indices:"), - // "Indices: {} {}, provided: {}", - // ranges.size(), - // cursor.offset_in_ranges, - // provided); auto & range = ranges[cursor.offset_in_ranges]; UInt128 can_provide = cursor.offset_in_ranges == end.offset_in_ranges @@ -286,14 +269,7 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - if (step == 1) - { - iota(pos, static_cast(size), start_value_64); - } - else - { - iota_with_step(pos, static_cast(size), start_value_64, step); - } + iota_with_step_optimized(pos, static_cast(size), start_value_64, step); pos += size; } }; @@ -302,14 +278,7 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - if (step == 1) - { - iota(pos, static_cast(need), start_value); - } - else - { - iota_with_step(pos, static_cast(need), start_value, step); - } + iota_with_step_optimized(pos, static_cast(need), start_value, step); pos += need; provided += need; cursor.offset_in_range += need; @@ -467,9 +436,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() Pipe pipe; Ranges ranges; - - // LOG_DEBUG(&Poco::Logger::get("parameters"), "Parameters: {} {} {} {}", numbers_storage.step, numbers_storage.offset, numbers_storage.limit.has_value(), numbers_storage.limit.has_value() ? numbers_storage.limit.value() : UInt64{0}); - if (numbers_storage.limit.has_value() && (numbers_storage.limit.value() == 0)) { pipe.addSource(std::make_shared(NumbersSource::createHeader(numbers_storage.column_name))); diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index d47c592c4b3..32d16ef38f9 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -40,9 +40,7 @@ void StorageSystemNumbers::read( size_t max_block_size, size_t num_streams) { - // LOG_DEBUG(&Poco::Logger::get("Reading from SystemNumbers"), "Limit : {}", limit.value()); query_plan.addStep(std::make_unique( column_names, query_info, storage_snapshot, context, shared_from_this(), max_block_size, num_streams)); } - } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 3fa55fe28c5..ada812c9f54 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -122,6 +122,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attachNoDescription(context, system_database, "numbers", "Generates all natural numbers, starting from 0 (to 2^64 - 1, and then again) in sorted order.", false, "number"); attachNoDescription(context, system_database, "numbers_mt", "Multithreaded version of `system.numbers`. Numbers order is not guaranteed.", true, "number"); attachNoDescription(context, system_database, "generate_series", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); + attachNoDescription(context, system_database, "generateSeries", "Generates arithmetic progression of natural numbers in sorted order in a given segment with a given step", false, "generate_series"); attachNoDescription(context, system_database, "zeros", "Produces unlimited number of non-materialized zeros.", false); attachNoDescription(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true); attach(context, system_database, "databases", "Lists all databases of the current server."); diff --git a/src/TableFunctions/TableFunctionGenerateSeries.cpp b/src/TableFunctions/TableFunctionGenerateSeries.cpp index 65c4c4915c2..ee833250d0b 100644 --- a/src/TableFunctions/TableFunctionGenerateSeries.cpp +++ b/src/TableFunctions/TableFunctionGenerateSeries.cpp @@ -24,14 +24,14 @@ namespace ErrorCodes namespace { -/* numbers(limit), numbers_mt(limit) - * - the same as SELECT number FROM system.numbers LIMIT limit. - * Used for testing purposes, as a simple example of table function. - */ +constexpr std::array names = {"generate_series", "generateSeries"}; + +template class TableFunctionGenerateSeries : public ITableFunction { public: - static constexpr auto name = "generate_series"; + static_assert(alias_num < names.size()); + static constexpr auto name = names[alias_num]; std::string getName() const override { return name; } bool hasStaticStructure() const override { return true; } @@ -49,13 +49,15 @@ private: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; }; -ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const +template +ColumnsDescription TableFunctionGenerateSeries::getActualTableStructure(ContextPtr /*context*/, bool /*is_insert_query*/) const { /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 return ColumnsDescription{{{"generate_series", std::make_shared()}}}; } -StoragePtr TableFunctionGenerateSeries::executeImpl( +template +StoragePtr TableFunctionGenerateSeries::executeImpl( const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, @@ -86,7 +88,8 @@ StoragePtr TableFunctionGenerateSeries::executeImpl( throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'limit' or 'offset, limit'.", getName()); } -UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const +template +UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr & argument) const { const auto & [field, type] = evaluateConstantExpression(argument, context); @@ -108,8 +111,8 @@ UInt64 TableFunctionGenerateSeries::evaluateArgument(ContextPtr context, ASTPtr void registerTableFunctionGenerateSeries(TableFunctionFactory & factory) { - factory.registerFunction({.documentation = {}, .allow_readonly = true}); - // factory.registerFunction({.documentation = {}, .allow_readonly = true}); + factory.registerFunction>({.documentation = {}, .allow_readonly = true}); + factory.registerFunction>({.documentation = {}, .allow_readonly = true}); } } diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index bcda8dc6a5e..2989eb5fbef 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include "base/types.h" #include "registerTableFunctions.h" @@ -68,15 +70,16 @@ StoragePtr TableFunctionNumbers::executeImpl( { auto arguments = function->arguments->children; - if (arguments.size() != 1 && arguments.size() != 2) + if ((arguments.empty()) || (arguments.size() >= 4)) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' requires 'length' or 'offset, length'.", getName()); - UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; - UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); + UInt64 offset = arguments.size() >= 2 ? evaluateArgument(context, arguments[0]) : 0; + UInt64 length = arguments.size() >= 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); + UInt64 step = arguments.size() == 3 ? evaluateArgument(context, arguments[2]) : 1; auto res = std::make_shared( - StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset); + StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset, step); res->startup(); return res; } From d0ed47e655913f66fec5ce31dbf10a2e045424ef Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:04:31 +0000 Subject: [PATCH 138/332] Return right cmake settings --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6847b1bf994..8c4e16eace2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,19 +54,19 @@ endif () # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # # set DATA (since RSS does not work since 2.6.x+) to 5G - # set (RLIMIT_DATA 5000000000) - # # set VIRT (RLIMIT_AS) to 10G (DATA*2) - # set (RLIMIT_AS 10000000000) - # # set CPU time limit to 1000 seconds - # set (RLIMIT_CPU 1000) + # set DATA (since RSS does not work since 2.6.x+) to 5G + set (RLIMIT_DATA 5000000000) + # set VIRT (RLIMIT_AS) to 10G (DATA*2) + set (RLIMIT_AS 10000000000) + # set CPU time limit to 1000 seconds + set (RLIMIT_CPU 1000) - # # -fsanitize=memory and address are too heavy - # if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) - # set (RLIMIT_DATA 10000000000) # 10G - # endif() + # -fsanitize=memory and address are too heavy + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) + set (RLIMIT_DATA 10000000000) # 10G + endif() - # set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 13b27a8d3663c6c160b826eef131601f226afacd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 18 Mar 2024 16:13:02 +0000 Subject: [PATCH 139/332] refactoring of reading from compact parts --- .../MergeTree/MergeTreeReaderCompact.cpp | 55 ++++++++++--------- .../MergeTree/MergeTreeReaderCompact.h | 3 +- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 3835c1e2054..a22bff6b8d2 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -78,31 +78,7 @@ void MergeTreeReaderCompact::fillColumnPositions() /// we have to read its offsets if they exist. if (!position && is_array) { - NameAndTypePair column_to_read_with_subcolumns = column_to_read; - auto [name_in_storage, subcolumn_name] = Nested::splitName(column_to_read.name); - - /// If it is a part of Nested, we need to get the column from - /// storage metadata which is converted to Nested type with subcolumns. - /// It is required for proper counting of shared streams. - if (!subcolumn_name.empty()) - { - /// If column is renamed get the new name from storage metadata. - if (alter_conversions->columnHasNewName(name_in_storage)) - name_in_storage = alter_conversions->getColumnNewName(name_in_storage); - - if (!storage_columns_with_collected_nested) - { - auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); - auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options)); - storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list)); - } - - column_to_read_with_subcolumns = storage_columns_with_collected_nested - ->getColumnOrSubcolumn( - GetColumnsOptions::All, - Nested::concatenateName(name_in_storage, subcolumn_name)); - } - + auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_to_read); auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns); if (name_level_for_offsets.has_value()) @@ -119,6 +95,35 @@ void MergeTreeReaderCompact::fillColumnPositions() } } +NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column) +{ + if (!isArray(column.type)) + return column; + + /// If it is a part of Nested, we need to get the column from + /// storage metadata which is converted to Nested type with subcolumns. + /// It is required for proper counting of shared streams. + auto [name_in_storage, subcolumn_name] = Nested::splitName(column.name); + + if (subcolumn_name.empty()) + return column; + + /// If column is renamed get the new name from storage metadata. + if (alter_conversions->columnHasNewName(name_in_storage)) + name_in_storage = alter_conversions->getColumnNewName(name_in_storage); + + if (!storage_columns_with_collected_nested) + { + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options)); + storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list)); + } + + return storage_columns_with_collected_nested->getColumnOrSubcolumn( + GetColumnsOptions::All, + Nested::concatenateName(name_in_storage, subcolumn_name)); +} + void MergeTreeReaderCompact::readData( const NameAndTypePair & name_and_type, ColumnPtr & column, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 9c66b4691a3..a783e595af5 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -31,10 +31,11 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_); - bool canReadIncompleteGranules() const override final { return false; } + bool canReadIncompleteGranules() const final { return false; } protected: void fillColumnPositions(); + NameAndTypePair getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column); using InputStreamGetter = ISerialization::InputStreamGetter; From 3fd5c6920579d927ee994646115211cae046c2e6 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:17:54 +0000 Subject: [PATCH 140/332] Add docs --- docs/en/sql-reference/table-functions/numbers.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/sql-reference/table-functions/numbers.md b/docs/en/sql-reference/table-functions/numbers.md index 7d3437b7d53..e0ff19b9824 100644 --- a/docs/en/sql-reference/table-functions/numbers.md +++ b/docs/en/sql-reference/table-functions/numbers.md @@ -8,6 +8,7 @@ sidebar_label: numbers `numbers(N)` – Returns a table with the single ‘number’ column (UInt64) that contains integers from 0 to N-1. `numbers(N, M)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1). +`numbers(N, M, S)` - Returns a table with the single ‘number’ column (UInt64) that contains integers from N to (N + M - 1) with step S. Similar to the `system.numbers` table, it can be used for testing and generating successive values, `numbers(N, M)` more efficient than `system.numbers`. @@ -21,6 +22,15 @@ SELECT * FROM system.numbers WHERE number BETWEEN 0 AND 9; SELECT * FROM system.numbers WHERE number IN (0, 1, 2, 3, 4, 5, 6, 7, 8, 9); ``` +And the following queries are equivalent: + +``` sql +SELECT number * 2 FROM numbers(10); +SELECT (number - 10) * 2 FROM numbers(10, 10); +SELECT * FROM numbers(0, 20, 2); +``` + + Examples: ``` sql From 2061fb25c5e88766bdce1f5a2ea9f0a734edbcf9 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:21:08 +0000 Subject: [PATCH 141/332] Add alias to doc --- docs/en/sql-reference/table-functions/generate_series.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index c5d29369627..9370cea8580 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -22,4 +22,7 @@ And the following queries return tables with the same content but different colu ``` sql SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); SELECT * FROM generate_series(10, 20, 3) ; -``` \ No newline at end of file +``` + +### Alias +generateSeries \ No newline at end of file From b814fabcd3224ab6904e5f8f947ca79a8f8ba02b Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 18 Mar 2024 17:18:01 +0100 Subject: [PATCH 142/332] Disallow s3queue sharded mode --- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + src/Storages/S3Queue/StorageS3Queue.cpp | 14 ++++++++++++-- src/Storages/S3Queue/StorageS3Queue.h | 3 ++- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 415063eee84..8508fc5d2d3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -114,6 +114,7 @@ class IColumn; M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ M(Bool, s3queue_enable_logging_to_s3queue_log, false, "Enable writing to system.s3queue_log. The value can be overwritten per table with table settings", 0) \ + M(Bool, s3queue_allow_experimental_sharded_mode, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten", 0) \ M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \ M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \ M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 072b9803682..ff5289230ca 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -100,6 +100,7 @@ static std::map sett {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, + {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 6e7ac2b47b8..048816fe4f1 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -105,7 +105,8 @@ StorageS3Queue::StorageS3Queue( const String & comment, ContextPtr context_, std::optional format_settings_, - ASTStorage * engine_args) + ASTStorage * engine_args, + LoadingStrictnessLevel mode) : IStorage(table_id_) , WithContext(context_) , s3queue_settings(std::move(s3queue_settings_)) @@ -129,6 +130,14 @@ StorageS3Queue::StorageS3Queue( throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue url must either end with '/' or contain globs"); } + if (mode == LoadingStrictnessLevel::CREATE + && !context_->getSettingsRef().s3queue_allow_experimental_sharded_mode + && s3queue_settings->mode == S3QueueMode::ORDERED + && s3queue_settings->s3queue_total_shards_num) + { + throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue sharded mode is not allowed. To enable use `s3queue_allow_experimental_sharded_mode`"); + } + checkAndAdjustSettings(*s3queue_settings, context_->getSettingsRef()); configuration.update(context_); @@ -663,7 +672,8 @@ void registerStorageS3QueueImpl(const String & name, StorageFactory & factory) args.comment, args.getContext(), format_settings, - args.storage_def); + args.storage_def, + args.mode); }, { .supports_settings = true, diff --git a/src/Storages/S3Queue/StorageS3Queue.h b/src/Storages/S3Queue/StorageS3Queue.h index bdd3ab7b687..33fccc20b84 100644 --- a/src/Storages/S3Queue/StorageS3Queue.h +++ b/src/Storages/S3Queue/StorageS3Queue.h @@ -37,7 +37,8 @@ public: const String & comment, ContextPtr context_, std::optional format_settings_, - ASTStorage * engine_args); + ASTStorage * engine_args, + LoadingStrictnessLevel mode); String getName() const override { return "S3Queue"; } From 4c27287c6d40f803121d2ef7263799c7460c4b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 18 Mar 2024 17:30:29 +0100 Subject: [PATCH 143/332] Fix crash in arrayPartialReverseSort --- src/Functions/array/arraySort.cpp | 5 ++++- ...013_fuzz_arrayPartialReverseSort.reference | 14 ++++++++++++++ .../03013_fuzz_arrayPartialReverseSort.sql | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.reference create mode 100644 tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.sql diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 184b1f82280..6c8741e6eec 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -46,7 +46,10 @@ ColumnPtr ArraySortImpl::execute( ErrorCodes::LOGICAL_ERROR, "Expected fixed arguments to get the limit for partial array sort" ); - return fixed_arguments[0].column.get()->getUInt(0); + + /// During dryRun the input column might be empty + if (!fixed_arguments[0].column->empty()) + return fixed_arguments[0].column->getUInt(0); } return 0; }(); diff --git a/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.reference b/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.reference new file mode 100644 index 00000000000..5832f8b7a52 --- /dev/null +++ b/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.reference @@ -0,0 +1,14 @@ +[] +[] +[1,0] +[] +[3,2] +[] +[5,4] +[] +[] +[1,0] +[] +[3,2] +[] +[5,4] diff --git a/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.sql b/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.sql new file mode 100644 index 00000000000..f575ae952dd --- /dev/null +++ b/tests/queries/0_stateless/03013_fuzz_arrayPartialReverseSort.sql @@ -0,0 +1,19 @@ +SELECT res +FROM +( + SELECT + arrayPartialReverseSort(2, if(number % 2, emptyArrayUInt64(), range(number))) AS arr, + arrayResize(arr, if(empty(arr), 0, 2)) AS res + FROM system.numbers + LIMIT 7 +); + +SELECT res +FROM +( + SELECT + arrayPartialReverseSort(materialize(2), if(number % 2, emptyArrayUInt64(), range(number))) AS arr, + arrayResize(arr, if(empty(arr), 0, 2)) AS res + FROM system.numbers + LIMIT 7 +); From 84051d8604ce6cbd03510cf77bda53a3f95382fa Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:34:16 +0100 Subject: [PATCH 144/332] Don't run 01459_manual_write_to_replicas in debug build as it's too slow --- tests/queries/0_stateless/01459_manual_write_to_replicas.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh index a9a6d27c145..56620d848a3 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: replica, no-parallel +# Tags: replica, no-parallel, no-debug set -e From 90c4b0aaf0e5ed0823b24d6c5ce48e726accf451 Mon Sep 17 00:00:00 2001 From: divanik Date: Mon, 18 Mar 2024 16:52:21 +0000 Subject: [PATCH 145/332] Add word to aspell-dictg --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 57a8e0d5840..edc3d993ed9 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1581,6 +1581,7 @@ gccMurmurHash gcem generateRandom generateRandomStructure +generateSeries generateULID generateUUIDv geoDistance From 613c31038ae76be09d35a151e6446024b07f127b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:40:03 +0100 Subject: [PATCH 146/332] Fix recompilation of regexps during backtracking in the parser --- src/Analyzer/MatcherNode.cpp | 23 +++++--- src/Analyzer/MatcherNode.h | 13 ++--- src/Analyzer/QueryTreeBuilder.cpp | 11 +--- .../JoinToSubqueryTransformVisitor.cpp | 12 +++- .../TranslateQualifiedNamesVisitor.cpp | 15 ++++- src/Parsers/ASTColumnsMatcher.cpp | 57 +++++-------------- src/Parsers/ASTColumnsMatcher.h | 18 ++---- src/Parsers/ExpressionElementParsers.cpp | 23 ++++---- 8 files changed, 73 insertions(+), 99 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index ae2b654ab13..d0efb0646b3 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -14,9 +13,15 @@ #include #include + namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_COMPILE_REGEXP; +} + const char * toString(MatcherNodeType matcher_node_type) { switch (matcher_node_type) @@ -48,20 +53,20 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, ColumnTransformersNod { } -MatcherNode::MatcherNode(std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, {} /*qualified_identifier*/, {} /*columns_identifiers*/, - std::move(columns_matcher_), + std::move(pattern_), std::move(column_transformers_)) { } -MatcherNode::MatcherNode(Identifier qualified_identifier_, std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, std::move(qualified_identifier_), {} /*columns_identifiers*/, - std::move(columns_matcher_), + std::move(pattern_), std::move(column_transformers_)) { } @@ -87,14 +92,18 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - std::shared_ptr columns_matcher_, + const String & pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) - , columns_matcher(columns_matcher_) { + columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + if (!columns_matcher->ok()) + throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + auto column_transformers_list_node = std::make_shared(); auto & column_transformers_nodes = column_transformers_list_node->getNodes(); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 87ee5ee92bb..2217fde6eff 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -58,10 +59,10 @@ public: explicit MatcherNode(Identifier qualified_identifier_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS('regexp') - explicit MatcherNode(std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant qualified COLUMNS('regexp') - explicit MatcherNode(Identifier qualified_identifier_, std::shared_ptr columns_matcher_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS(column_name_1, ...) explicit MatcherNode(Identifiers columns_identifiers_, ColumnTransformersNodes column_transformers_ = {}); @@ -81,12 +82,6 @@ public: return matcher_type == MatcherNodeType::ASTERISK; } - /// Returns true if matcher is columns regexp or columns list matcher, false otherwise - bool isColumnsMatcher() const - { - return matcher_type == MatcherNodeType::COLUMNS_REGEXP || matcher_type == MatcherNodeType::COLUMNS_LIST; - } - /// Returns true if matcher is qualified, false otherwise bool isQualified() const { @@ -152,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - std::shared_ptr columns_matcher_, + const String & pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 14559cca887..df80f46b3cd 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -2,11 +2,7 @@ #include -#include -#include -#include #include -#include #include #include #include @@ -33,13 +29,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include #include @@ -50,7 +44,6 @@ #include #include -#include namespace DB @@ -661,7 +654,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co else if (const auto * columns_regexp_matcher = expression->as()) { auto column_transformers = buildColumnTransformers(columns_regexp_matcher->transformers, context); - result = std::make_shared(columns_regexp_matcher->getMatcher(), std::move(column_transformers)); + result = std::make_shared(columns_regexp_matcher->getPattern(), std::move(column_transformers)); } else if (const auto * columns_list_matcher = expression->as()) { @@ -681,7 +674,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co { auto & qualified_identifier = qualified_columns_regexp_matcher->qualifier->as(); auto column_transformers = buildColumnTransformers(qualified_columns_regexp_matcher->transformers, context); - result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers)); + result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getPattern(), std::move(column_transformers)); } else if (const auto * qualified_columns_list_matcher = expression->as()) { diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 5c4ae528fc1..209f8e68b8f 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -15,11 +15,10 @@ #include #include #include -#include #include -#include #include #include +#include namespace DB { @@ -30,6 +29,7 @@ namespace ErrorCodes extern const int AMBIGUOUS_COLUMN_NAME; extern const int NOT_IMPLEMENTED; extern const int UNKNOWN_IDENTIFIER; + extern const int CANNOT_COMPILE_REGEXP; } namespace @@ -204,11 +204,17 @@ private: { has_asterisks = true; + String pattern = columns_regexp_matcher->getPattern(); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + for (auto & table_name : data.tables_order) data.addTableColumns( table_name, columns, - [&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); }); + [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, *regexp); }); if (columns_regexp_matcher->transformers) { diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 3de7e217e53..e3690d0fcc0 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -6,7 +5,6 @@ #include #include -#include #include #include @@ -21,6 +19,7 @@ #include #include #include +#include namespace DB @@ -31,7 +30,10 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; + extern const int CANNOT_COMPILE_REGEXP; } + + bool TranslateQualifiedNamesMatcher::Data::matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type) { if (name.size() < column_name.size()) @@ -275,12 +277,19 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt } else if (const auto * asterisk_regexp_pattern = child->as()) { + String pattern = asterisk_regexp_pattern->getPattern(); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + bool first_table = true; for (const auto & table : tables_with_columns) { for (const auto & column : table.columns) { - if (asterisk_regexp_pattern->isColumnMatching(column.name) && (first_table || !data.join_using_columns.contains(column.name))) + if (re2::RE2::PartialMatch(column.name, *regexp) + && (first_table || !data.join_using_columns.contains(column.name))) { addIdentifier(columns, table.table, column.name); } diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index c2854e2235c..dac3661d482 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -34,14 +34,14 @@ void ASTColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const writeCString(".", ostr); } writeCString("COLUMNS(", ostr); - writeQuotedString(original_pattern, ostr); + writeQuotedString(pattern, ostr); writeChar(')', ostr); } void ASTColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -56,7 +56,7 @@ void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, Format } settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(original_pattern); + settings.ostr << quoteString(pattern); settings.ostr << ")"; if (transformers) @@ -65,28 +65,14 @@ void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, Format } } -void ASTColumnsRegexpMatcher::setPattern(String pattern) +void ASTColumnsRegexpMatcher::setPattern(String pattern_) { - original_pattern = std::move(pattern); - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } const String & ASTColumnsRegexpMatcher::getPattern() const { - return original_pattern; -} - -const std::shared_ptr & ASTColumnsRegexpMatcher::getMatcher() const -{ - return column_matcher; -} - -bool ASTColumnsRegexpMatcher::isColumnMatching(const String & column_name) const -{ - return RE2::PartialMatch(column_name, *column_matcher); + return pattern; } ASTPtr ASTColumnsListMatcher::clone() const @@ -166,37 +152,24 @@ void ASTQualifiedColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) cons { qualifier->appendColumnName(ostr); writeCString(".COLUMNS(", ostr); - writeQuotedString(original_pattern, ostr); + writeQuotedString(pattern, ostr); writeChar(')', ostr); } -void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern, bool set_matcher) +void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern_) { - original_pattern = std::move(pattern); - - if (!set_matcher) - return; - - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } -void ASTQualifiedColumnsRegexpMatcher::setMatcher(std::shared_ptr matcher) +const String & ASTQualifiedColumnsRegexpMatcher::getPattern() const { - column_matcher = std::move(matcher); -} - -const std::shared_ptr & ASTQualifiedColumnsRegexpMatcher::getMatcher() const -{ - return column_matcher; + return pattern; } void ASTQualifiedColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -207,7 +180,7 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting qualifier->formatImpl(settings, state, frame); settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "("; - settings.ostr << quoteString(original_pattern); + settings.ostr << quoteString(pattern); settings.ostr << ")"; if (transformers) diff --git a/src/Parsers/ASTColumnsMatcher.h b/src/Parsers/ASTColumnsMatcher.h index 6fc5581a4eb..e5191b74b72 100644 --- a/src/Parsers/ASTColumnsMatcher.h +++ b/src/Parsers/ASTColumnsMatcher.h @@ -2,11 +2,6 @@ #include -namespace re2 -{ -class RE2; -} - namespace DB { @@ -25,8 +20,6 @@ public: void appendColumnName(WriteBuffer & ostr) const override; void setPattern(String pattern); const String & getPattern() const; - const std::shared_ptr & getMatcher() const; - bool isColumnMatching(const String & column_name) const; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; ASTPtr expression; @@ -35,8 +28,7 @@ protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; /// Same as the above but use a list of column names to do matching. @@ -62,9 +54,8 @@ public: ASTPtr clone() const override; void appendColumnName(WriteBuffer & ostr) const override; - const std::shared_ptr & getMatcher() const; - void setPattern(String pattern, bool set_matcher = true); - void setMatcher(std::shared_ptr matcher); + void setPattern(String pattern_); + const String & getPattern() const; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; ASTPtr qualifier; @@ -73,8 +64,7 @@ protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; private: - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; /// Same as ASTColumnsListMatcher. Qualified identifier is first child. diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 1135fe88c16..67f4a306292 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include @@ -44,6 +43,7 @@ #include + namespace DB { @@ -1628,7 +1628,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e is_strict = true; ASTs identifiers; - ASTPtr regex_node; + ASTPtr regexp_node; ParserStringLiteral regex; auto parse_id = [&identifiers, &pos, &expected] { @@ -1644,7 +1644,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e { // support one or more parameter ++pos; - if (!ParserList::parseUtil(pos, expected, parse_id, false) && !regex.parse(pos, regex_node, expected)) + if (!ParserList::parseUtil(pos, expected, parse_id, false) && !regex.parse(pos, regexp_node, expected)) return false; if (pos->type != TokenType::ClosingRoundBracket) @@ -1654,13 +1654,13 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e else { // only one parameter - if (!parse_id() && !regex.parse(pos, regex_node, expected)) + if (!parse_id() && !regex.parse(pos, regexp_node, expected)) return false; } auto res = std::make_shared(); - if (regex_node) - res->setPattern(regex_node->as().value.get()); + if (regexp_node) + res->setPattern(regexp_node->as().value.get()); else res->children = std::move(identifiers); res->is_strict = is_strict; @@ -1794,11 +1794,11 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected ++pos; ParserList columns_p(std::make_unique(false, true), std::make_unique(TokenType::Comma), false); - ParserStringLiteral regex; + ParserStringLiteral regexp; ASTPtr column_list; - ASTPtr regex_node; - if (!columns_p.parse(pos, column_list, expected) && !regex.parse(pos, regex_node, expected)) + ASTPtr regexp_node; + if (!columns_p.parse(pos, column_list, expected) && !regexp.parse(pos, regexp_node, expected)) return false; if (pos->type != TokenType::ClosingRoundBracket) @@ -1832,7 +1832,7 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected else { auto regexp_matcher = std::make_shared(); - regexp_matcher->setPattern(regex_node->as().value.get()); + regexp_matcher->setPattern(regexp_node->as().value.get()); if (!transformers->children.empty()) { @@ -1895,8 +1895,7 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected else if (auto * column_regexp_matcher = node->as()) { auto result = std::make_shared(); - result->setPattern(column_regexp_matcher->getPattern(), false); - result->setMatcher(column_regexp_matcher->getMatcher()); + result->setPattern(column_regexp_matcher->getPattern()); result->qualifier = std::move(identifier_node); result->children.push_back(result->qualifier); From 05ee8369b298cb7091ac621bee4098f801353f3f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:54:13 +0100 Subject: [PATCH 147/332] Add a test --- .../03013_parser_regexp_recompilation.reference | 1 + .../0_stateless/03013_parser_regexp_recompilation.sh | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03013_parser_regexp_recompilation.reference create mode 100755 tests/queries/0_stateless/03013_parser_regexp_recompilation.sh diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference b/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference new file mode 100644 index 00000000000..8e0782c44f2 --- /dev/null +++ b/tests/queries/0_stateless/03013_parser_regexp_recompilation.reference @@ -0,0 +1 @@ +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh b/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh new file mode 100755 index 00000000000..ea7a72ce73d --- /dev/null +++ b/tests/queries/0_stateless/03013_parser_regexp_recompilation.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# This query is incorrect, but its parsing should not be slow (tens of seconds): + +${CLICKHOUSE_LOCAL} < "$CURDIR"/03013_parser_regexp_recompilation.txt 2>&1 | grep --text -o -F 'SYNTAX_ERROR' From 597b72dc5221fdcd2d4e87f24489d0d164a90527 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 18 Mar 2024 18:54:28 +0100 Subject: [PATCH 148/332] Add a test --- .../03013_parser_regexp_recompilation.txt | Bin 0 -> 55948 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/queries/0_stateless/03013_parser_regexp_recompilation.txt diff --git a/tests/queries/0_stateless/03013_parser_regexp_recompilation.txt b/tests/queries/0_stateless/03013_parser_regexp_recompilation.txt new file mode 100644 index 0000000000000000000000000000000000000000..a861059c3a1322eae212bca0f4b5fc2e1be7de1e GIT binary patch literal 55948 zcmeHQUymEN5tmcMD1^38MNyy#ut5Re-R5lEAEQYc7{SF>f& z=)70Fm`;8eT#kp6qIde;@oBI4)6vPv_Ty(K&!3*2Z*Sf9yX-2vzv|w;yVb64_W@Ml zbP%?m zABexEgk}%Akip624<;s*YKZacMd!E4FYaVM;@YvUzVjX4_?VUPq5V9^DnOwpI)+RG zua*EL$5?Nt>y%O;3zr^JdBGB>IG_Pw*~Mx~sO7wlEGup9Ezkv*?g!pg)mgOKpq`vO zTRV<9tF?|2TrM*o<22_>-x~xPsLQC+Zc*r7bzu>wny~){+f-WaVUb3KhPz+O!Us>p zGaHA2jCg(OSHF)deYK7W=K|ELS?!&9<-6GwpXw}#ikSctLv-;q4yi6<+%0*twzlTvP|SC zYWkj&V~SJGjIg|$hlsN-e~gwO%N@%oiPSjE&z?OkUXLcD*(dYS*>Li5dbzz*oE$$r z?iYt&b&pTaduRRP__Y75m=6Z8uHU{L>=oy|liuTgQ5N5-*28e?-J0e%gZa&y(N*!N zn9r_br^)Xff}c9dXwhi57)^%XUyKGX$CKg3cz$2&8mq4dZ>Cpc+R)F>$5WFFbFC9s&g^CVk$4DuLu9Q9?iyLg!lLM?(gjvdwYN6|84CSThIHCw|0u7^J2bR z%!|*8eMY8F7^vIxo#HbnVJ_&M{rTwRudeIrXmV41fav_9&lG*X|5dm9SbVx*A1+RM zP*iMBis#3Z`QgECF)WT|v*FG5w-?jl{0|Rzb{I0R$CJ^cr=!d1>_$1??|WC6{LK|6 zZ=1wPO_)SPSxsxr(|&(?YD8=Z~szPz!Cy@7L^M@G^mbI*D09~^x7`47a2+3&IMGMM6cM_aENV$+U zO#(MkYj_qx4{27QRC-D<3G83kN``)A{Xl2%Cw`&ThAClANm`AX4P=?fQPkp&2d(xd z*A4{ra1gx_)8Zq-nb4=W{L`=m0q$5vi6wEEtHUyz@s;S)z-)_rQ@G%-0QpGP!Vdnq zKGMohy!l9${Lz4XX3u@Dv|WG%G7qV)l2HCk1gC+Qk9(ufM0K0?pWbI8Jd3TJe91?$ z>{IF=JgW1NEc)*WnxDuzbf3O*mgXZ_WPOutwanF~+q~D#N3x9I$w#vCzmG9O;(c$c zB$W5sYp~a@bMsGcuT9@Y=ObC{Ehl@!S+YO$H7DHqU#t3SN@zx2m%?p88iF(8rETz4 zt>`MyS+r?EJ^2@HT04$;thJ63+|B|>Uj8Uy0@C@8KJ$?*l>{8{8DB3)J#5mCa_)I;9x`W&p74 zz5+kR#}+2JB{c1&_onTnuI}p;5o>TrO>1S`I1SOJN#I86Th&=)3|HLZpjEz$IGr5r z7U%*y5R`WsfKjU!R(6tO`ZZUq8wYphyg^(vNoiD5f;vzqqgdpkrYXMe%(g_Xk*7Kx z*h!Msl6C6WoD{f`dXoE4=rzr=eP|EK-IB;TU?8=TtY%qV8x-^fMqOJ{7gB@uqDlTI z0t7nFB0fk-efoG{opAkuxuaD}gEn#^WrRt%SuBPmQX#42oW0gg z+?KPp6jUqarEh7G;?eI-R*2St0lj0#NV!D>q97}W&@gAoRt|!OASEK}JYXD-qI_Hn z2rs<@fnGFl(SpP)7OO0f3>OH-5fP9Y8-|qGtyq3n4Pgr{S&)-72>^|yh%Mg7v^O7b zTEXQsn*%peaSm_`VJEagDmphs3tGLC0;eGXw`$c=*Q)xM47Fl$&FakMM|H zL*THPPL)T1(W~@36!Zm#F73Swslj@oAV0#JKq5X!NqzcwV4ZON&OM~3P#Tnc5fplH z*cDx72FeJNaI@ITCxRmat$|y!M|d)H!b*Ug6NiXd=NH5zEU5_RO~4zUBD6F@0cvfM z=v(`jc8msa>zrVD3V8$XUn^zfs-{OLdgzd~jfmuE$L`FXaI!*)s56df8$@1FUYRq2 z6F5>RG5lbD$C;x11`Ks<10HptPDZiFMVGlB5~IjZWa!QGOAKmuQS`nqFkEdMWg#_K zFPi2jG800?2Pvsf9}lb(uAk((AS(?@o(zRv9Ck&QnSnCGB-|{v@`>PxKx^RE=ZOr) zuKxf5?7vF8n3cd<#s~nV{j!X|rkNftZE0_+Lu%>D%V%^mwZS3g->qrwIOed{I=ENW zBZMBYpxnM>mxSvl~st*h9a`s8D_TvfRc)E-jZ97f77R%%}1i8p1_TSA$ews zec^}efHn-vH*p2119dWrMJ~Gh)g{09K}V)fD5%**(fhu@aJ6xih16iZXqsRAm=Gd9 zNJ)MAcwn7y{ek?aRZD}CCqtnZhh5QSW}u8P2{((ad?Gj^&>Fb)dGW(X?Sw*`CV?A? z-Kx$a;@XPEK{r_;S_cMnj~yfBCItZ2s^u6OqAgoF2pWQvh^+H~aX5q>2i6JK?@D^UGHFoqWGM9Fuq&Eh7Wq8!fxh=B z*Q5;1yiN0YnV_sEbMOPNEU5_Rm5UV9a28n_p#YUNSs^w%XT@&doN!i%I$clOAo7ax z%ItS1aHNo8X!DyLn_|2wVic1z0YfJF?j>ge2rzq&IsZ==Tw?WQDtaZurO8AOvQqsA z^E Date: Mon, 18 Mar 2024 15:18:47 -0300 Subject: [PATCH 149/332] fix typo in variable name legacy --- src/Storages/MergeTree/MergeTreePartInfo.cpp | 10 +++++----- src/Storages/MergeTree/MergeTreePartInfo.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.cpp b/src/Storages/MergeTree/MergeTreePartInfo.cpp index 8c1b630e85e..f3a974a22af 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.cpp +++ b/src/Storages/MergeTree/MergeTreePartInfo.cpp @@ -131,7 +131,7 @@ std::optional MergeTreePartInfo::tryParsePartName( /// "Part 20170601_20170630_0_2_999999999 intersects 201706_0_1_4294967295". /// So we replace unexpected max level to make contains(...) method and comparison operators work /// correctly with such virtual parts. On part name serialization we will use legacy max level to keep the name unchanged. - part_info.use_leagcy_max_level = true; + part_info.use_legacy_max_level = true; level = MAX_LEVEL; } @@ -205,7 +205,7 @@ String MergeTreePartInfo::getPartNameV1() const writeChar('_', wb); writeIntText(max_block, wb); writeChar('_', wb); - if (use_leagcy_max_level) + if (use_legacy_max_level) { assert(level == MAX_LEVEL); writeIntText(LEGACY_MAX_LEVEL, wb); @@ -244,7 +244,7 @@ String MergeTreePartInfo::getPartNameV0(DayNum left_date, DayNum right_date) con writeChar('_', wb); writeIntText(max_block, wb); writeChar('_', wb); - if (use_leagcy_max_level) + if (use_legacy_max_level) { assert(level == MAX_LEVEL); writeIntText(LEGACY_MAX_LEVEL, wb); @@ -274,7 +274,7 @@ void MergeTreePartInfo::serialize(WriteBuffer & out) const writeIntBinary(max_block, out); writeIntBinary(level, out); writeIntBinary(mutation, out); - writeBoolText(use_leagcy_max_level, out); + writeBoolText(use_legacy_max_level, out); } @@ -297,7 +297,7 @@ void MergeTreePartInfo::deserialize(ReadBuffer & in) readIntBinary(max_block, in); readIntBinary(level, in); readIntBinary(mutation, in); - readBoolText(use_leagcy_max_level, in); + readBoolText(use_legacy_max_level, in); } bool MergeTreePartInfo::areAllBlockNumbersCovered(const MergeTreePartInfo & blocks_range, std::vector candidates) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 5fbb5d70bf3..9bb79e21144 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -26,7 +26,7 @@ struct MergeTreePartInfo UInt32 level = 0; Int64 mutation = 0; /// If the part has been mutated or contains mutated parts, is equal to mutation version number. - bool use_leagcy_max_level = false; /// For compatibility. TODO remove it + bool use_legacy_max_level = false; /// For compatibility. TODO remove it MergeTreePartInfo() = default; From 9af69b42668c9c4363af1865e86d04453ad5ddcc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 18 Mar 2024 21:05:09 +0100 Subject: [PATCH 150/332] fix read_rows count with external group by --- src/Processors/ISource.h | 2 +- src/Processors/Transforms/AggregatingTransform.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Processors/ISource.h b/src/Processors/ISource.h index 8c140d0d0a3..778042c5b56 100644 --- a/src/Processors/ISource.h +++ b/src/Processors/ISource.h @@ -44,7 +44,7 @@ public: void setStorageLimits(const std::shared_ptr & storage_limits_) override; /// Default implementation for all the sources. - std::optional getReadProgress() final; + std::optional getReadProgress() override; void addTotalRowsApprox(size_t value); void addTotalBytes(size_t value); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 74da97f2199..ea5c525d5f2 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -80,6 +80,8 @@ namespace return convertToChunk(block); } + std::optional getReadProgress() override { return std::nullopt; } + private: TemporaryFileStream * tmp_stream; }; From 306f642d97ce51bed6fda59cac1c3e3465f6ef5a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 17 Mar 2024 11:57:01 +0100 Subject: [PATCH 151/332] Update dictionary's comment in the same safe manner too. --- src/Dictionaries/DictionaryFactory.cpp | 6 +----- src/Dictionaries/IDictionary.h | 14 ++++++++------ .../ExternalDictionariesLoader.cpp | 13 +++++++++---- src/Storages/StorageDictionary.cpp | 18 +++++++++--------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index a566fb27de4..337121822ea 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -55,11 +55,7 @@ DictionaryPtr DictionaryFactory::create( if (found != registered_layouts.end()) { const auto & layout_creator = found->second.layout_create_function; - auto result = layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); - if (config.hasProperty(config_prefix + ".comment")) - result->setDictionaryComment(config.getString(config_prefix + ".comment")); - - return result; + return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); } } diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index 9770e93669e..bab80d3cd57 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -332,12 +332,6 @@ public: return std::static_pointer_cast(IExternalLoadable::shared_from_this()); } - void setDictionaryComment(String new_comment) - { - std::lock_guard lock{mutex}; - dictionary_comment = std::move(new_comment); - } - String getDictionaryComment() const { std::lock_guard lock{mutex}; @@ -454,6 +448,14 @@ public: dictionary_id = new_dictionary_id; } + /// Internally called by ExternalDictionariesLoader. + /// In order to update the dictionary comment change its configuration first and then call ExternalDictionariesLoader::reloadConfig(). + void updateDictionaryComment(const String & new_dictionary_comment) + { + std::lock_guard lock{mutex}; + dictionary_comment = new_dictionary_comment; + } + private: mutable std::mutex mutex; StorageID dictionary_id TSA_GUARDED_BY(mutex); diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 0591a6232d0..68f22df2947 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -45,14 +45,17 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const { + std::unordered_set ignore_keys; + ignore_keys.insert("comment"); /// We always can change the comment without reloading a dictionary. + /// If the database is atomic then a dictionary can be renamed without reloading. if (!old_config.getString(old_key_in_config + ".uuid", "").empty() && !new_config.getString(new_key_in_config + ".uuid", "").empty()) { - static const std::unordered_set ignore_keys{"name", "database"}; - bool only_name_or_database_may_differ = isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); - return !only_name_or_database_may_differ; + ignore_keys.insert("name"); + ignore_keys.insert("database"); } - return true; + + return !isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); } void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const @@ -67,6 +70,8 @@ void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExterna if ((new_dictionary_id.uuid == old_dictionary_id.uuid) && (new_dictionary_id.uuid != UUIDHelpers::Nil)) dict.updateDictionaryID(new_dictionary_id); } + + dict.updateDictionaryComment(config.getString(key_in_config + ".comment", "")); } ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 414c66e9522..a0c4156a704 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -278,19 +278,19 @@ void StorageDictionary::alter(const AlterCommands & params, ContextPtr alter_con auto new_comment = getInMemoryMetadataPtr()->comment; - auto storage_id = getStorageID(); - const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); - auto result = external_dictionaries_loader.getLoadResult(storage_id.getInternalDictionaryName()); + /// It's better not to update an associated `IDictionary` directly here because it can be not loaded yet or + /// it can be in the process of loading or reloading right now. + /// The correct way is to update the dictionary's configuration first and then ask ExternalDictionariesLoader to reload our dictionary. - if (result.object) { - auto dictionary = std::static_pointer_cast(result.object); - auto * dictionary_non_const = const_cast(dictionary.get()); - dictionary_non_const->setDictionaryComment(new_comment); + std::lock_guard lock(dictionary_config_mutex); + auto new_configuration = ConfigHelper::clone(*configuration); + new_configuration->setString("dictionary.comment", new_comment); + configuration = new_configuration; } - std::lock_guard lock(dictionary_config_mutex); - configuration->setString("dictionary.comment", new_comment); + const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); } void registerStorageDictionary(StorageFactory & factory) From c8375cd16777010a35d11340bf7b8c9c60c1fa1e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 18 Mar 2024 16:44:55 +0100 Subject: [PATCH 152/332] Better code: remove unnecessary usage of const_cast and std::function. --- src/Dictionaries/CacheDictionary.h | 2 +- src/Dictionaries/DirectDictionary.h | 2 +- src/Dictionaries/FlatDictionary.h | 2 +- src/Dictionaries/HashedArrayDictionary.h | 2 +- src/Dictionaries/HashedDictionary.h | 2 +- src/Dictionaries/IPAddressDictionary.h | 2 +- .../PolygonDictionaryImplementations.cpp | 6 +- .../PolygonDictionaryImplementations.h | 6 +- src/Dictionaries/RangeHashedDictionary.h | 2 +- src/Dictionaries/RegExpTreeDictionary.h | 2 +- ...alUserDefinedExecutableFunctionsLoader.cpp | 2 +- ...rnalUserDefinedExecutableFunctionsLoader.h | 2 +- .../UserDefinedExecutableFunction.h | 2 +- .../ExternalDictionariesLoader.cpp | 2 +- src/Interpreters/ExternalDictionariesLoader.h | 4 +- src/Interpreters/ExternalLoader.cpp | 60 +++++-------------- src/Interpreters/ExternalLoader.h | 7 ++- src/Interpreters/IExternalLoadable.h | 2 +- 18 files changed, 41 insertions(+), 68 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 8897fb40fa9..50f59e3eddb 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -98,7 +98,7 @@ public: bool supportUpdates() const override { return false; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared( getDictionaryID(), diff --git a/src/Dictionaries/DirectDictionary.h b/src/Dictionaries/DirectDictionary.h index 8bf183fd1ab..d4b4cd8e698 100644 --- a/src/Dictionaries/DirectDictionary.h +++ b/src/Dictionaries/DirectDictionary.h @@ -50,7 +50,7 @@ public: double getLoadFactor() const override { return 0; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone()); } diff --git a/src/Dictionaries/FlatDictionary.h b/src/Dictionaries/FlatDictionary.h index 231819ab4ee..7b00ce57455 100644 --- a/src/Dictionaries/FlatDictionary.h +++ b/src/Dictionaries/FlatDictionary.h @@ -57,7 +57,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), configuration, update_field_loaded_block); } diff --git a/src/Dictionaries/HashedArrayDictionary.h b/src/Dictionaries/HashedArrayDictionary.h index 4b2570ad928..d4bb67b9606 100644 --- a/src/Dictionaries/HashedArrayDictionary.h +++ b/src/Dictionaries/HashedArrayDictionary.h @@ -73,7 +73,7 @@ public: double getLoadFactor() const override { return static_cast(total_element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared>(getDictionaryID(), dict_struct, source_ptr->clone(), configuration, update_field_loaded_block); } diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index 37812de29d9..4080c018289 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -116,7 +116,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared>( getDictionaryID(), diff --git a/src/Dictionaries/IPAddressDictionary.h b/src/Dictionaries/IPAddressDictionary.h index cebdefcf7ae..bdd02157077 100644 --- a/src/Dictionaries/IPAddressDictionary.h +++ b/src/Dictionaries/IPAddressDictionary.h @@ -57,7 +57,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), configuration); } diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 3feca2ec410..64d29458430 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -29,7 +29,7 @@ PolygonDictionarySimple::PolygonDictionarySimple( { } -std::shared_ptr PolygonDictionarySimple::clone() const +std::shared_ptr PolygonDictionarySimple::clone() const { return std::make_shared( this->getDictionaryID(), @@ -76,7 +76,7 @@ PolygonDictionaryIndexEach::PolygonDictionaryIndexEach( } } -std::shared_ptr PolygonDictionaryIndexEach::clone() const +std::shared_ptr PolygonDictionaryIndexEach::clone() const { return std::make_shared( this->getDictionaryID(), @@ -126,7 +126,7 @@ PolygonDictionaryIndexCell::PolygonDictionaryIndexCell( { } -std::shared_ptr PolygonDictionaryIndexCell::clone() const +std::shared_ptr PolygonDictionaryIndexCell::clone() const { return std::make_shared( this->getDictionaryID(), diff --git a/src/Dictionaries/PolygonDictionaryImplementations.h b/src/Dictionaries/PolygonDictionaryImplementations.h index 912d501bcde..690ff3a0f1b 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.h +++ b/src/Dictionaries/PolygonDictionaryImplementations.h @@ -23,7 +23,7 @@ public: DictionaryLifetime dict_lifetime_, Configuration configuration_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; private: bool find(const Point & point, size_t & polygon_index) const override; @@ -47,7 +47,7 @@ public: int min_intersections_, int max_depth_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; static constexpr size_t kMinIntersectionsDefault = 1; static constexpr size_t kMaxDepthDefault = 5; @@ -75,7 +75,7 @@ public: size_t min_intersections_, size_t max_depth_); - std::shared_ptr clone() const override; + std::shared_ptr clone() const override; static constexpr size_t kMinIntersectionsDefault = 1; static constexpr size_t kMaxDepthDefault = 5; diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index 1a6ee7e81d4..9f7dc5dc68a 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -101,7 +101,7 @@ public: double getLoadFactor() const override { return static_cast(element_count) / bucket_count; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { auto result = std::make_shared( getDictionaryID(), diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 9e14abb49d0..a361c68879e 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -86,7 +86,7 @@ public: bool hasHierarchy() const override { return false; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared( getDictionaryID(), structure, source_ptr->clone(), configuration, use_vectorscan, flag_case_insensitive, flag_dotall); diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp index db98f88e53b..a4f17aa1201 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -120,7 +120,7 @@ void ExternalUserDefinedExecutableFunctionsLoader::reloadFunction(const std::str loadOrReload(user_defined_function_name); } -ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create(const std::string & name, +ExternalLoader::LoadableMutablePtr ExternalUserDefinedExecutableFunctionsLoader::createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string &) const diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h index 1a62175eb0c..eb86986c391 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h @@ -28,7 +28,7 @@ public: void reloadFunction(const std::string & user_defined_function_name) const; protected: - LoadablePtr create(const std::string & name, + LoadableMutablePtr createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const override; diff --git a/src/Functions/UserDefined/UserDefinedExecutableFunction.h b/src/Functions/UserDefined/UserDefinedExecutableFunction.h index 989f9dfe895..d48be215c7d 100644 --- a/src/Functions/UserDefined/UserDefinedExecutableFunction.h +++ b/src/Functions/UserDefined/UserDefinedExecutableFunction.h @@ -62,7 +62,7 @@ public: return true; } - std::shared_ptr clone() const override + std::shared_ptr clone() const override { return std::make_shared(configuration, coordinator, lifetime); } diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 68f22df2947..34ad240d089 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -32,7 +32,7 @@ ExternalDictionariesLoader::ExternalDictionariesLoader(ContextPtr global_context enablePeriodicUpdates(true); } -ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( +ExternalLoader::LoadableMutablePtr ExternalDictionariesLoader::createObject( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const { diff --git a/src/Interpreters/ExternalDictionariesLoader.h b/src/Interpreters/ExternalDictionariesLoader.h index f3aa8fa5f8e..0fc58aa1edf 100644 --- a/src/Interpreters/ExternalDictionariesLoader.h +++ b/src/Interpreters/ExternalDictionariesLoader.h @@ -40,8 +40,8 @@ public: static void resetAll(); protected: - LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, - const std::string & key_in_config, const std::string & repository_name) const override; + LoadableMutablePtr createObject(const std::string & name, const Poco::Util::AbstractConfiguration & config, + const std::string & key_in_config, const std::string & repository_name) const override; bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & old_config, const String & old_key_in_config, const Poco::Util::AbstractConfiguration & new_config, const String & new_key_in_config) const override; diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index 6ab083f9fb7..bd56a540128 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -87,9 +87,6 @@ namespace lock = std::unique_lock(mutex); } }; - - using DoesConfigChangeRequiresReloadingObjectFunction = std::function; - using UpdateObjectFromConfigWithoutReloadingFunction = std::function; } @@ -98,10 +95,7 @@ namespace class ExternalLoader::LoadablesConfigReader : private boost::noncopyable { public: - LoadablesConfigReader(const String & type_name_, LoggerPtr log_) - : type_name(type_name_), log(log_) - { - } + LoadablesConfigReader(const String & type_name_, LoggerPtr log_) : type_name(type_name_), log(log_) { } ~LoadablesConfigReader() = default; using Repository = IExternalLoaderConfigRepository; @@ -397,21 +391,8 @@ private: class ExternalLoader::LoadingDispatcher : private boost::noncopyable { public: - /// Called to load or reload an object. - using CreateObjectFunction = std::function; - - LoadingDispatcher( - const CreateObjectFunction & create_object_function_, - const DoesConfigChangeRequiresReloadingObjectFunction & does_config_change_requires_reloading_object_, - const UpdateObjectFromConfigWithoutReloadingFunction & update_object_from_config_without_reloading_, - const String & type_name_, - LoggerPtr log_) - : create_object(create_object_function_) - , does_config_change_requires_reloading_object(does_config_change_requires_reloading_object_) - , update_object_from_config_without_reloading(update_object_from_config_without_reloading_) - , type_name(type_name_) - , log(log_) + LoadingDispatcher(const String & type_name_, LoggerPtr log_, const ExternalLoader & external_loader_) + : type_name(type_name_), log(log_), external_loader(external_loader_) { } @@ -471,11 +452,11 @@ public: if (config_changed) { if (info.object) - update_object_from_config_without_reloading(*info.object, *new_config->config, new_config->key_in_config); + external_loader.updateObjectFromConfigWithoutReloading(*info.object, *new_config->config, new_config->key_in_config); if (info.triedToLoad()) { - bool config_change_requires_reloading = does_config_change_requires_reloading_object(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); + bool config_change_requires_reloading = external_loader.doesConfigChangeRequiresReloadingObject(*previous_config->config, previous_config->key_in_config, *new_config->config, new_config->key_in_config); if (config_change_requires_reloading) { /// The object has been tried to load before, so it is currently in use or was in use @@ -786,7 +767,7 @@ private: } String name; - LoadablePtr object; + LoadableMutablePtr object; std::shared_ptr config; TimePoint loading_start_time; TimePoint loading_end_time; @@ -1046,17 +1027,17 @@ private: } /// Load one object, returns object ptr or exception. - std::pair + std::pair loadSingleObject(const String & name, const ObjectConfig & config, LoadablePtr previous_version) { /// Use `create_function` to perform the actual loading. /// It's much better to do it with `mutex` unlocked because the loading can take a lot of time /// and require access to other objects. - LoadablePtr new_object; + LoadableMutablePtr new_object; std::exception_ptr new_exception; try { - new_object = create_object(name, config, previous_version); + new_object = external_loader.createOrCloneObject(name, config, previous_version); } catch (...) { @@ -1070,7 +1051,7 @@ private: const String & name, size_t loading_id, LoadablePtr previous_version, - LoadablePtr new_object, + LoadableMutablePtr new_object, std::exception_ptr new_exception, size_t error_count, const LoadingGuardForAsyncLoad &) @@ -1134,7 +1115,7 @@ private: if (new_object) { - update_object_from_config_without_reloading(*new_object, *info->config->config, info->config->key_in_config); + external_loader.updateObjectFromConfigWithoutReloading(*new_object, *info->config->config, info->config->key_in_config); info->object = new_object; } @@ -1210,11 +1191,9 @@ private: } } - const CreateObjectFunction create_object; - const DoesConfigChangeRequiresReloadingObjectFunction does_config_change_requires_reloading_object; - const UpdateObjectFromConfigWithoutReloadingFunction update_object_from_config_without_reloading; const String type_name; - LoggerPtr log; + const LoggerPtr log; + const ExternalLoader & external_loader; mutable std::mutex mutex; std::condition_variable event; @@ -1296,14 +1275,7 @@ private: ExternalLoader::ExternalLoader(const String & type_name_, LoggerPtr log_) : config_files_reader(std::make_unique(type_name_, log_)) - , loading_dispatcher(std::make_unique( - [this](auto && a, auto && b, auto && c) { return createObject(a, b, c); }, - [this](const Poco::Util::AbstractConfiguration & config_1, const String & key_in_config_1, const Poco::Util::AbstractConfiguration & config_2, const String & key_in_config_2) - { return doesConfigChangeRequiresReloadingObject(config_1, key_in_config_1, config_2, key_in_config_2); }, - [this](const IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) - { return updateObjectFromConfigWithoutReloading(const_cast(object), config, key_in_config); }, - type_name_, - log_)) + , loading_dispatcher(std::make_unique(type_name_, log_, *this)) , periodic_updater(std::make_unique(*config_files_reader, *loading_dispatcher)) , type_name(type_name_) , log(log_) @@ -1530,13 +1502,13 @@ void ExternalLoader::reloadConfig(const String & repository_name, const String & loading_dispatcher->setConfiguration(config_files_reader->read(repository_name, path)); } -ExternalLoader::LoadablePtr ExternalLoader::createObject( +ExternalLoader::LoadableMutablePtr ExternalLoader::createOrCloneObject( const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const { if (previous_version) return previous_version->clone(); - return create(name, *config.config, config.key_in_config, config.repository_name); + return createObject(name, *config.config, config.key_in_config, config.repository_name); } template ExternalLoader::LoadablePtr ExternalLoader::getLoadResult(const String &) const; diff --git a/src/Interpreters/ExternalLoader.h b/src/Interpreters/ExternalLoader.h index 781d8ca9e7b..c746ce39b2d 100644 --- a/src/Interpreters/ExternalLoader.h +++ b/src/Interpreters/ExternalLoader.h @@ -50,6 +50,7 @@ class ExternalLoader { public: using LoadablePtr = std::shared_ptr; + using LoadableMutablePtr = std::shared_ptr; using Loadables = std::vector; using Status = ExternalLoaderStatus; @@ -211,7 +212,7 @@ public: void reloadConfig(const String & repository_name, const String & path) const; protected: - virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; + virtual LoadableMutablePtr createObject(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; /// Returns whether the object must be reloaded after a specified change in its configuration. virtual bool doesConfigChangeRequiresReloadingObject(const Poco::Util::AbstractConfiguration & /* old_config */, const String & /* old_key_in_config */, @@ -227,7 +228,7 @@ private: Strings getAllTriedToLoadNames() const; - LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; + LoadableMutablePtr createOrCloneObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; class LoadablesConfigReader; std::unique_ptr config_files_reader; @@ -239,7 +240,7 @@ private: std::unique_ptr periodic_updater; const String type_name; - LoggerPtr log; + const LoggerPtr log; }; } diff --git a/src/Interpreters/IExternalLoadable.h b/src/Interpreters/IExternalLoadable.h index 3c004508b0a..1076bd88027 100644 --- a/src/Interpreters/IExternalLoadable.h +++ b/src/Interpreters/IExternalLoadable.h @@ -43,7 +43,7 @@ public: /// If lifetime exceeded and isModified(), ExternalLoader replace current object with the result of clone(). virtual bool isModified() const = 0; /// Returns new object with the same configuration. Is used to update modified object when lifetime exceeded. - virtual std::shared_ptr clone() const = 0; + virtual std::shared_ptr clone() const = 0; }; } From a1a4bd3514f72b4545fbb734749dcb9d7f513007 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Wed, 6 Mar 2024 10:42:30 +0800 Subject: [PATCH 153/332] make nulls direction configuable for FullSortingMergeJoin --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 7 +- src/Processors/QueryPlan/JoinStep.cpp | 8 +- src/Processors/QueryPlan/JoinStep.h | 4 +- .../Transforms/MergeJoinTransform.cpp | 82 ++++++++++++------- .../Transforms/MergeJoinTransform.h | 16 +++- src/QueryPipeline/QueryPipelineBuilder.cpp | 3 +- src/QueryPipeline/QueryPipelineBuilder.h | 1 + 8 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a3c5638d97f..8d48b3f5e68 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,6 +891,7 @@ class IColumn; M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ + M(Bool, nulls_biggest_in_smj, true, "Treat nulls as biggest in sort. Used in sort merge join for compare null keys.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7c87dadfce6..6f0a9fa9bfb 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1693,9 +1693,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( query_plan.getCurrentDataStream(), joined_plan->getCurrentDataStream(), expressions.join, settings.max_block_size, max_streams, - analysis_result.optimize_read_in_order); + analysis_result.optimize_read_in_order, + null_direct_hint); join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType())); std::vector plans; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 1931b1eb3a1..0c46ce7893d 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -44,8 +44,10 @@ JoinStep::JoinStep( JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_) - : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_) + bool keep_left_read_in_order_, + int null_direction_hint_) + : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_), + null_direction_hint(null_direction_hint_) { updateInputStreams(DataStreams{left_stream_, right_stream_}); } @@ -58,7 +60,7 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines if (join->pipelineType() == JoinPipelineType::YShaped) { auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped( - std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, &processors); + std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, null_direction_hint, &processors); joined_pipeline->resize(max_streams); return joined_pipeline; } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index a9059a083fe..08909ce48a9 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -19,7 +19,8 @@ public: JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_); + bool keep_left_read_in_order_, + int null_direction_hint_ = 1); String getName() const override { return "Join"; } @@ -42,6 +43,7 @@ private: size_t max_block_size; size_t max_streams; bool keep_left_read_in_order; + int null_direction_hint; }; /// Special step for the case when Join is already filled. diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 2d313d4ea5c..c8e3a806a9f 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -43,7 +43,7 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns) } template -int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint = 1) +int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint) { if constexpr (has_left_nulls && has_right_nulls) { @@ -88,35 +88,36 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, } int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos, - const SortCursorImpl & rhs, size_t rpos) + const SortCursorImpl & rhs, size_t rpos, + int null_direction_hint) { for (size_t i = 0; i < lhs.sort_columns_size; ++i) { /// TODO(@vdimir): use nullableCompareAt only if there's nullable columns - int cmp = nullableCompareAt(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos); + int cmp = nullableCompareAt(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos, null_direction_hint); if (cmp != 0) return cmp; } return 0; } -int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs) +int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs, int null_direction_hint) { - return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow()); + return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), null_direction_hint); } -bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs) +bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) { /// The last row of left cursor is less than the current row of the right cursor. - int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow()); + int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), null_direction_hint); return cmp < 0; } -int ALWAYS_INLINE totallyCompare(SortCursorImpl & lhs, SortCursorImpl & rhs) +int ALWAYS_INLINE totallyCompare(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint) { - if (totallyLess(lhs, rhs)) + if (totallyLess(lhs, rhs, null_direction_hint)) return -1; - if (totallyLess(rhs, lhs)) + if (totallyLess(rhs, lhs, null_direction_hint)) return 1; return 0; } @@ -270,9 +271,11 @@ bool FullMergeJoinCursor::fullyCompleted() const MergeJoinAlgorithm::MergeJoinAlgorithm( JoinPtr table_join_, const Blocks & input_headers, - size_t max_block_size_) + size_t max_block_size_, + int null_direction_hint_) : table_join(table_join_) , max_block_size(max_block_size_) + , null_direction_hint(null_direction_hint_) , log(getLogger("MergeJoinAlgorithm")) { if (input_headers.size() != 2) @@ -356,7 +359,7 @@ void MergeJoinAlgorithm::consume(Input & input, size_t source_num) cursors[source_num]->setChunk(std::move(input.chunk)); } -template +template struct AllJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind); @@ -382,7 +385,7 @@ struct AllJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(nullOrder)); if (cmp == 0) { size_t lnum = nextDistinct(left_cursor.cursor); @@ -432,19 +435,37 @@ struct AllJoinImpl } }; -template class Impl, typename ... Args> -void dispatchKind(JoinKind kind, Args && ... args) +template class Impl, typename ... Args> +void dispatchKind(JoinKind kind, int null_direction_hint, Args && ... args) { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); + if (isSmall(null_direction_hint)) + { + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + + } else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + { + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + + } } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -517,7 +538,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) { PaddedPODArray idx_map[2]; - dispatchKind(kind, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); + dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); assert(idx_map[0].size() == idx_map[1].size()); Chunk result; @@ -567,7 +588,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) } -template +template struct AnyJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind); @@ -599,7 +620,7 @@ struct AnyJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(order)); if (cmp == 0) { if constexpr (isLeftOrFull(kind)) @@ -723,7 +744,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind) PaddedPODArray idx_map[2]; size_t prev_pos[] = {current_left.getRow(), current_right.getRow()}; - dispatchKind(kind, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); + dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); assert(idx_map[0].empty() || idx_map[1].empty() || idx_map[0].size() == idx_map[1].size()); size_t num_result_rows = std::max(idx_map[0].size(), idx_map[1].size()); @@ -816,7 +837,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() } /// check if blocks are not intersecting at all - if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor); cmp != 0) + if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0) { if (cmp < 0) { @@ -851,6 +872,7 @@ MergeJoinTransform::MergeJoinTransform( const Blocks & input_headers, const Block & output_header, size_t max_block_size, + int null_direction_hint_, UInt64 limit_hint_) : IMergingTransform( input_headers, @@ -859,7 +881,7 @@ MergeJoinTransform::MergeJoinTransform( limit_hint_, /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, - table_join, input_headers, max_block_size) + table_join, input_headers, max_block_size, null_direction_hint_) , log(getLogger("MergeJoinTransform")) { LOG_TRACE(log, "Use MergeJoinTransform"); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 959550067f7..43485321122 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -220,6 +220,17 @@ private: bool recieved_all_blocks = false; }; +/// Join method. +enum class NullOrder +{ + SMALLEST, /// null is treated as smallest + BIGGEST /// null is treated as biggest +}; + +inline constexpr bool isSmall(int null_direction) { return null_direction == 1; } + +inline constexpr int nullDirection(NullOrder order) {return order == NullOrder::SMALLEST ? 1 : -1;} + /* * This class is used to join chunks from two sorted streams. * It is used in MergeJoinTransform. @@ -227,7 +238,8 @@ private: class MergeJoinAlgorithm final : public IMergingAlgorithm { public: - explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_); + explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_, + int null_direction_hint = 1); const char * getName() const override { return "MergeJoinAlgorithm"; } void initialize(Inputs inputs) override; @@ -258,6 +270,7 @@ private: JoinPtr table_join; size_t max_block_size; + int null_direction_hint; struct Statistic { @@ -282,6 +295,7 @@ public: const Blocks & input_headers, const Block & output_header, size_t max_block_size, + int null_direction_hint, UInt64 limit_hint = 0); String getName() const override { return "MergeJoinTransform"; } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 67a8fe5dcab..e338c3ce0fa 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -349,6 +349,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped JoinPtr join, const Block & out_header, size_t max_block_size, + int null_direction_hint, Processors * collected_processors) { left->checkInitializedAndNotCompleted(); @@ -376,7 +377,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped } else { - auto joining = std::make_shared(join, inputs, out_header, max_block_size); + auto joining = std::make_shared(join, inputs, out_header, max_block_size, null_direction_hint); return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors); } } diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index f0b2ead687e..4753f957a25 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -137,6 +137,7 @@ public: JoinPtr table_join, const Block & out_header, size_t max_block_size, + int null_direction_hint, Processors * collected_processors = nullptr); /// Add other pipeline and execute it before current one. From 0b5fc743f2e0711556ab4628aecd13e5fcd1a9b8 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 12 Mar 2024 09:55:02 +0800 Subject: [PATCH 154/332] make nulls direction configuable for FullSortingMergeJoin(fix review comments) --- src/Core/Settings.h | 1 - src/Interpreters/InterpreterSelectQuery.cpp | 7 +-- .../Transforms/MergeJoinTransform.cpp | 56 +++++++------------ .../Transforms/MergeJoinTransform.h | 11 ---- 4 files changed, 22 insertions(+), 53 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8d48b3f5e68..a3c5638d97f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,7 +891,6 @@ class IColumn; M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ - M(Bool, nulls_biggest_in_smj, true, "Treat nulls as biggest in sort. Used in sort merge join for compare null keys.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6f0a9fa9bfb..7c87dadfce6 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1693,10 +1693,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( query_plan.getCurrentDataStream(), joined_plan->getCurrentDataStream(), expressions.join, settings.max_block_size, max_streams, - analysis_result.optimize_read_in_order, - null_direct_hint); + analysis_result.optimize_read_in_order); join_step->setStepDescription(fmt::format("JOIN {}", expressions.join->pipelineType())); std::vector plans; diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index c8e3a806a9f..6288a850d76 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -359,7 +359,7 @@ void MergeJoinAlgorithm::consume(Input & input, size_t source_num) cursors[source_num]->setChunk(std::move(input.chunk)); } -template +template struct AllJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind); @@ -369,7 +369,8 @@ struct AllJoinImpl size_t max_block_size, PaddedPODArray & left_map, PaddedPODArray & right_map, - std::unique_ptr & state) + std::unique_ptr & state, + int null_direction_hint) { right_map.clear(); right_map.reserve(max_block_size); @@ -385,7 +386,7 @@ struct AllJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(nullOrder)); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, null_direction_hint); if (cmp == 0) { size_t lnum = nextDistinct(left_cursor.cursor); @@ -435,37 +436,19 @@ struct AllJoinImpl } }; -template class Impl, typename ... Args> -void dispatchKind(JoinKind kind, int null_direction_hint, Args && ... args) +template class Impl, typename ... Args> +void dispatchKind(JoinKind kind, Args && ... args) { - if (isSmall(null_direction_hint)) - { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); + if (Impl::enabled && kind == JoinKind::Inner) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); - - } - else - { - if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); - else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); - - } } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -538,7 +521,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) { PaddedPODArray idx_map[2]; - dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state); + dispatchKind(kind, *cursors[0], *cursors[1], max_block_size, idx_map[0], idx_map[1], all_join_state, null_direction_hint); assert(idx_map[0].size() == idx_map[1].size()); Chunk result; @@ -588,7 +571,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind) } -template +template struct AnyJoinImpl { constexpr static bool enabled = isInner(kind) || isLeft(kind) || isRight(kind); @@ -597,7 +580,8 @@ struct AnyJoinImpl FullMergeJoinCursor & right_cursor, PaddedPODArray & left_map, PaddedPODArray & right_map, - AnyJoinState & state) + AnyJoinState & state, + int null_direction_hint) { assert(enabled); @@ -620,7 +604,7 @@ struct AnyJoinImpl lpos = left_cursor->getRow(); rpos = right_cursor->getRow(); - cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, nullDirection(order)); + cmp = compareCursors(left_cursor.cursor, right_cursor.cursor, null_direction_hint); if (cmp == 0) { if constexpr (isLeftOrFull(kind)) @@ -744,7 +728,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind) PaddedPODArray idx_map[2]; size_t prev_pos[] = {current_left.getRow(), current_right.getRow()}; - dispatchKind(kind, null_direction_hint, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state); + dispatchKind(kind, *cursors[0], *cursors[1], idx_map[0], idx_map[1], any_join_state, null_direction_hint); assert(idx_map[0].empty() || idx_map[1].empty() || idx_map[0].size() == idx_map[1].size()); size_t num_result_rows = std::max(idx_map[0].size(), idx_map[1].size()); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 43485321122..8af486ea34b 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -220,17 +220,6 @@ private: bool recieved_all_blocks = false; }; -/// Join method. -enum class NullOrder -{ - SMALLEST, /// null is treated as smallest - BIGGEST /// null is treated as biggest -}; - -inline constexpr bool isSmall(int null_direction) { return null_direction == 1; } - -inline constexpr int nullDirection(NullOrder order) {return order == NullOrder::SMALLEST ? 1 : -1;} - /* * This class is used to join chunks from two sorted streams. * It is used in MergeJoinTransform. From 5cf22bae6f40ab1beb258cf22a9b0627c601495d Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 12 Mar 2024 10:04:37 +0800 Subject: [PATCH 155/332] minor: fix style change --- .../Transforms/MergeJoinTransform.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 6288a850d76..37a178810cb 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -440,15 +440,15 @@ template class Impl, typename ... Args> void dispatchKind(JoinKind kind, Args && ... args) { if (Impl::enabled && kind == JoinKind::Inner) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Left) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Right) - return Impl::join(std::forward(args)...); - else if (Impl::enabled && kind == JoinKind::Full) - return Impl::join(std::forward(args)...); - else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Left) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Right) + return Impl::join(std::forward(args)...); + else if (Impl::enabled && kind == JoinKind::Full) + return Impl::join(std::forward(args)...); + else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); } std::optional MergeJoinAlgorithm::handleAllJoinState() From 8d49ff7350304619386cb849851432a5ca434619 Mon Sep 17 00:00:00 2001 From: Dan Wu Date: Tue, 19 Mar 2024 03:12:32 +0000 Subject: [PATCH 156/332] Add tests for system.parts_columns table. --- .../03010_read_system_parts_table_test.reference | 10 ++++++++-- .../0_stateless/03010_read_system_parts_table_test.sql | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference index 6847786761e..c2dd177f245 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.reference +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.reference @@ -1,4 +1,10 @@ 00000000-0000-0000-0000-000000000000 1231_1_1_0 00000000-0000-0000-0000-000000000000 6666_2_2_0 -users -users +00000000-0000-0000-0000-000000000000 1231_1_1_0 users +00000000-0000-0000-0000-000000000000 6666_2_2_0 users +00000000-0000-0000-0000-000000000000 1231_1_1_0 users uid +00000000-0000-0000-0000-000000000000 1231_1_1_0 users name +00000000-0000-0000-0000-000000000000 1231_1_1_0 users age +00000000-0000-0000-0000-000000000000 6666_2_2_0 users uid +00000000-0000-0000-0000-000000000000 6666_2_2_0 users name +00000000-0000-0000-0000-000000000000 6666_2_2_0 users age diff --git a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql index 8871822af4e..a50005b2312 100644 --- a/tests/queries/0_stateless/03010_read_system_parts_table_test.sql +++ b/tests/queries/0_stateless/03010_read_system_parts_table_test.sql @@ -6,5 +6,6 @@ INSERT INTO users VALUES (6666, 'Ksenia', 48); SELECT uuid, name from system.parts WHERE database = currentDatabase() AND table = 'users'; -SELECT table from system.parts WHERE database = currentDatabase() AND uuid = '00000000-0000-0000-0000-000000000000'; +SELECT uuid, name, table from system.parts WHERE database = currentDatabase() AND table = 'users' AND uuid = '00000000-0000-0000-0000-000000000000'; +SELECT uuid, name, table, column from system.parts_columns WHERE database = currentDatabase() AND table = 'users' AND uuid = '00000000-0000-0000-0000-000000000000'; DROP TABLE IF EXISTS users; From 8ff21d7e47c25637b87f3ab674421cfc0a2c4487 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Tue, 19 Mar 2024 10:59:32 +0800 Subject: [PATCH 157/332] fix review comments --- src/Interpreters/FullSortingMergeJoin.h | 7 ++++++- src/Processors/QueryPlan/JoinStep.cpp | 8 +++----- src/Processors/QueryPlan/JoinStep.h | 4 +--- .../Transforms/MergeJoinTransform.cpp | 17 +++++++++++------ src/Processors/Transforms/MergeJoinTransform.h | 6 ++---- src/QueryPipeline/QueryPipelineBuilder.cpp | 3 +-- src/QueryPipeline/QueryPipelineBuilder.h | 1 - 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h index 7688d44f7a9..7e07c2004b6 100644 --- a/src/Interpreters/FullSortingMergeJoin.h +++ b/src/Interpreters/FullSortingMergeJoin.h @@ -21,9 +21,11 @@ namespace ErrorCodes class FullSortingMergeJoin : public IJoin { public: - explicit FullSortingMergeJoin(std::shared_ptr table_join_, const Block & right_sample_block_) + explicit FullSortingMergeJoin(std::shared_ptr table_join_, const Block & right_sample_block_, + int null_direction_ = 1) : table_join(table_join_) , right_sample_block(right_sample_block_) + , null_direction(null_direction_) { LOG_TRACE(getLogger("FullSortingMergeJoin"), "Will use full sorting merge join"); } @@ -31,6 +33,8 @@ public: std::string getName() const override { return "FullSortingMergeJoin"; } const TableJoin & getTableJoin() const override { return *table_join; } + int getNullDirection() const { return null_direction; } + bool addBlockToJoin(const Block & /* block */, bool /* check_limits */) override { throw Exception(ErrorCodes::LOGICAL_ERROR, "FullSortingMergeJoin::addBlockToJoin should not be called"); @@ -119,6 +123,7 @@ private: std::shared_ptr table_join; Block right_sample_block; Block totals; + int null_direction; }; } diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 0c46ce7893d..1931b1eb3a1 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -44,10 +44,8 @@ JoinStep::JoinStep( JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_, - int null_direction_hint_) - : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_), - null_direction_hint(null_direction_hint_) + bool keep_left_read_in_order_) + : join(std::move(join_)), max_block_size(max_block_size_), max_streams(max_streams_), keep_left_read_in_order(keep_left_read_in_order_) { updateInputStreams(DataStreams{left_stream_, right_stream_}); } @@ -60,7 +58,7 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines if (join->pipelineType() == JoinPipelineType::YShaped) { auto joined_pipeline = QueryPipelineBuilder::joinPipelinesYShaped( - std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, null_direction_hint, &processors); + std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, &processors); joined_pipeline->resize(max_streams); return joined_pipeline; } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index 08909ce48a9..a9059a083fe 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -19,8 +19,7 @@ public: JoinPtr join_, size_t max_block_size_, size_t max_streams_, - bool keep_left_read_in_order_, - int null_direction_hint_ = 1); + bool keep_left_read_in_order_); String getName() const override { return "Join"; } @@ -43,7 +42,6 @@ private: size_t max_block_size; size_t max_streams; bool keep_left_read_in_order; - int null_direction_hint; }; /// Special step for the case when Join is already filled. diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 37a178810cb..b63598483ef 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -271,11 +272,9 @@ bool FullMergeJoinCursor::fullyCompleted() const MergeJoinAlgorithm::MergeJoinAlgorithm( JoinPtr table_join_, const Blocks & input_headers, - size_t max_block_size_, - int null_direction_hint_) + size_t max_block_size_) : table_join(table_join_) , max_block_size(max_block_size_) - , null_direction_hint(null_direction_hint_) , log(getLogger("MergeJoinAlgorithm")) { if (input_headers.size() != 2) @@ -305,6 +304,13 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( size_t right_idx = input_headers[1].getPositionByName(right_key); left_to_right_key_remap[left_idx] = right_idx; } + + auto smjPtr = typeid_cast(table_join.get()); + if (smjPtr) + { + null_direction_hint = smjPtr->getNullDirection(); + } + } void MergeJoinAlgorithm::logElapsed(double seconds) @@ -448,7 +454,7 @@ void dispatchKind(JoinKind kind, Args && ... args) else if (Impl::enabled && kind == JoinKind::Full) return Impl::join(std::forward(args)...); else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind); } std::optional MergeJoinAlgorithm::handleAllJoinState() @@ -856,7 +862,6 @@ MergeJoinTransform::MergeJoinTransform( const Blocks & input_headers, const Block & output_header, size_t max_block_size, - int null_direction_hint_, UInt64 limit_hint_) : IMergingTransform( input_headers, @@ -865,7 +870,7 @@ MergeJoinTransform::MergeJoinTransform( limit_hint_, /* always_read_till_end_= */ false, /* empty_chunk_on_finish_= */ true, - table_join, input_headers, max_block_size, null_direction_hint_) + table_join, input_headers, max_block_size) , log(getLogger("MergeJoinTransform")) { LOG_TRACE(log, "Use MergeJoinTransform"); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 8af486ea34b..cf9331abd59 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -227,8 +227,7 @@ private: class MergeJoinAlgorithm final : public IMergingAlgorithm { public: - explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_, - int null_direction_hint = 1); + explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_); const char * getName() const override { return "MergeJoinAlgorithm"; } void initialize(Inputs inputs) override; @@ -259,7 +258,7 @@ private: JoinPtr table_join; size_t max_block_size; - int null_direction_hint; + int null_direction_hint = 1; struct Statistic { @@ -284,7 +283,6 @@ public: const Blocks & input_headers, const Block & output_header, size_t max_block_size, - int null_direction_hint, UInt64 limit_hint = 0); String getName() const override { return "MergeJoinTransform"; } diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index e338c3ce0fa..67a8fe5dcab 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -349,7 +349,6 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped JoinPtr join, const Block & out_header, size_t max_block_size, - int null_direction_hint, Processors * collected_processors) { left->checkInitializedAndNotCompleted(); @@ -377,7 +376,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesYShaped } else { - auto joining = std::make_shared(join, inputs, out_header, max_block_size, null_direction_hint); + auto joining = std::make_shared(join, inputs, out_header, max_block_size); return mergePipelines(std::move(left), std::move(right), std::move(joining), collected_processors); } } diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 4753f957a25..f0b2ead687e 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -137,7 +137,6 @@ public: JoinPtr table_join, const Block & out_header, size_t max_block_size, - int null_direction_hint, Processors * collected_processors = nullptr); /// Add other pipeline and execute it before current one. From 24a59546c7247c596627f708754d7fc7e6a08093 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Mar 2024 09:42:40 +0000 Subject: [PATCH 158/332] ASdd documentation --- docs/en/sql-reference/table-functions/generateSeries.md | 8 ++++++++ docs/en/sql-reference/table-functions/generate_series.md | 5 +---- ...4_all_new_table_functions_must_be_documented.reference | 1 + .../0_stateless/02992_all_columns_should_have_comment.sql | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/generateSeries.md diff --git a/docs/en/sql-reference/table-functions/generateSeries.md b/docs/en/sql-reference/table-functions/generateSeries.md new file mode 100644 index 00000000000..9dd2e09b6a7 --- /dev/null +++ b/docs/en/sql-reference/table-functions/generateSeries.md @@ -0,0 +1,8 @@ +--- +slug: /en/sql-reference/table-functions/generateSeries +sidebar_position: 147 +sidebar_label: generateSeries +--- + +### AliasTo +[generate_series](generate_series.md) \ No newline at end of file diff --git a/docs/en/sql-reference/table-functions/generate_series.md b/docs/en/sql-reference/table-functions/generate_series.md index 9370cea8580..c5d29369627 100644 --- a/docs/en/sql-reference/table-functions/generate_series.md +++ b/docs/en/sql-reference/table-functions/generate_series.md @@ -22,7 +22,4 @@ And the following queries return tables with the same content but different colu ``` sql SELECT * FROM numbers(10, 11) WHERE number % 3 == (10 % 3); SELECT * FROM generate_series(10, 20, 3) ; -``` - -### Alias -generateSeries \ No newline at end of file +``` \ No newline at end of file diff --git a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference index 1e4f21a6722..57ffeca9d43 100644 --- a/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02414_all_new_table_functions_must_be_documented.reference @@ -4,6 +4,7 @@ dictionary executable file generateRandom +generateSeries generate_series input jdbc diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql index b90faeac359..e233f08cc79 100644 --- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql +++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql @@ -1,4 +1,4 @@ SYSTEM FLUSH LOGS; SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || ' should have a comment' FROM system.columns -WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series')) AND (default_kind != 'ALIAS'); +WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries')) AND (default_kind != 'ALIAS'); From 87e2c09febe17ce45e29d7d02397193f23b250cc Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 19 Mar 2024 10:31:05 +0000 Subject: [PATCH 159/332] Correct typo --- docs/en/sql-reference/table-functions/generateSeries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/table-functions/generateSeries.md b/docs/en/sql-reference/table-functions/generateSeries.md index 9dd2e09b6a7..addf539a959 100644 --- a/docs/en/sql-reference/table-functions/generateSeries.md +++ b/docs/en/sql-reference/table-functions/generateSeries.md @@ -4,5 +4,5 @@ sidebar_position: 147 sidebar_label: generateSeries --- -### AliasTo +### Alias To [generate_series](generate_series.md) \ No newline at end of file From 12cd566681143d017b9b4fbfc3ff5d11af7b4f1a Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 11:47:53 +0100 Subject: [PATCH 160/332] Update config for test --- tests/integration/test_storage_s3_queue/configs/users.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_s3_queue/configs/users.xml b/tests/integration/test_storage_s3_queue/configs/users.xml index 1c096c7da8f..e47df8e4b94 100644 --- a/tests/integration/test_storage_s3_queue/configs/users.xml +++ b/tests/integration/test_storage_s3_queue/configs/users.xml @@ -3,6 +3,7 @@ 1 1 + 1 From eb75926e50d03ddba7327c3377b11ead3e221c80 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 21 Feb 2024 14:33:01 +0100 Subject: [PATCH 161/332] Parallel flush of pending INSERT blocks of Distributed engine Parallelism will work only if you have multi disk policy for table (like everything in Distributed engine right now). This will work for DETACH/server shutdown and SYSTEM FLUSH DISTRIBUTED Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3b766ac8d26..6554af1f7e6 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1734,16 +1734,27 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) directory_queues.push_back(node.second.directory_queue); } - bool need_flush = getDistributedSettingsRef().flush_on_detach; - if (!need_flush) + if (getDistributedSettingsRef().flush_on_detach) + { + LOG_INFO(log, "Flushing pending INSERT blocks"); + + Stopwatch watch; + ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); + for (const auto & node : directory_queues) + { + pool.scheduleOrThrowOnError([node_to_flush = node]() + { + node_to_flush->flushAllData(); + }); + } + pool.wait(); + LOG_INFO(log, "Pending INSERT blocks flushed, took {} ms.", watch.elapsedMilliseconds()); + } + else + { LOG_INFO(log, "Skip flushing data (due to flush_on_detach=0)"); - /// TODO: Maybe it should be executed in parallel - for (auto & node : directory_queues) - { - if (need_flush) - node->flushAllData(); - else + for (auto & node : directory_queues) node->shutdownWithoutFlush(); } } From 929dc6fa12ced13309fea73229a0c5213cdc0f30 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 22 Feb 2024 15:41:19 +0100 Subject: [PATCH 162/332] Attach to query for threads in Distributed engine background ops Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 6554af1f7e6..e25e3425359 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1287,10 +1287,19 @@ void StorageDistributed::initializeFromDisk() /// Make initialization for large number of disks parallel. ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); + ThreadGroupPtr thread_group = CurrentThread::getGroup(); for (const DiskPtr & disk : disks) { pool.scheduleOrThrowOnError([&]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistInit"); + initializeDirectoryQueuesForDisk(disk); }); } @@ -1302,6 +1311,14 @@ void StorageDistributed::initializeFromDisk() { pool.scheduleOrThrowOnError([&, i]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistInit"); + last_increment[i] = getMaximumFileNumber(paths[i]); }); } @@ -1739,11 +1756,20 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) LOG_INFO(log, "Flushing pending INSERT blocks"); Stopwatch watch; + ThreadGroupPtr thread_group = CurrentThread::getGroup(); ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); for (const auto & node : directory_queues) { - pool.scheduleOrThrowOnError([node_to_flush = node]() + pool.scheduleOrThrowOnError([node_to_flush = node, &thread_group]() { + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachToGroup(thread_group); + setThreadName("DistFlush"); + node_to_flush->flushAllData(); }); } From 8b1df937a567cec4acc6702288d979d4489bb83f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 Mar 2024 07:09:28 -0400 Subject: [PATCH 163/332] Use scheduleFromThreadPool for parallel processing in distributed storage Signed-off-by: Azat Khuzhin --- src/Storages/StorageDistributed.cpp | 63 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e25e3425359..7819aa7683a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -23,6 +23,7 @@ #include +#include #include #include #include @@ -282,6 +283,17 @@ size_t getClusterQueriedNodes(const Settings & settings, const ClusterPtr & clus return (num_remote_shards + num_local_shards) * settings.max_parallel_replicas; } +template +void waitFutures(F & futures) +{ + for (auto & future : futures) + future.wait(); + /// Make sure there is no exception. + for (auto & future : futures) + future.get(); + futures.clear(); +} + } /// For destruction of std::unique_ptr of type that is incomplete in class definition. @@ -1286,42 +1298,30 @@ void StorageDistributed::initializeFromDisk() /// Make initialization for large number of disks parallel. ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); + std::vector> futures; - ThreadGroupPtr thread_group = CurrentThread::getGroup(); for (const DiskPtr & disk : disks) { - pool.scheduleOrThrowOnError([&]() + auto future = scheduleFromThreadPool([this, disk_to_init = disk] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistInit"); - - initializeDirectoryQueuesForDisk(disk); - }); + initializeDirectoryQueuesForDisk(disk_to_init); + }, pool, "DistInit"); + futures.push_back(std::move(future)); } + waitFutures(futures); pool.wait(); const auto & paths = getDataPaths(); std::vector last_increment(paths.size()); for (size_t i = 0; i < paths.size(); ++i) { - pool.scheduleOrThrowOnError([&, i]() + auto future = scheduleFromThreadPool([&paths, &last_increment, i] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistInit"); - last_increment[i] = getMaximumFileNumber(paths[i]); - }); + }, pool, "DistInit"); + futures.push_back(std::move(future)); } + waitFutures(futures); pool.wait(); for (const auto inc : last_increment) @@ -1756,24 +1756,21 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context) LOG_INFO(log, "Flushing pending INSERT blocks"); Stopwatch watch; - ThreadGroupPtr thread_group = CurrentThread::getGroup(); ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, directory_queues.size()); + std::vector> futures; + for (const auto & node : directory_queues) { - pool.scheduleOrThrowOnError([node_to_flush = node, &thread_group]() + auto future = scheduleFromThreadPool([node_to_flush = node] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroup(thread_group); - setThreadName("DistFlush"); - node_to_flush->flushAllData(); - }); + }, pool, "DistFlush"); + futures.push_back(std::move(future)); } + + waitFutures(futures); pool.wait(); + LOG_INFO(log, "Pending INSERT blocks flushed, took {} ms.", watch.elapsedMilliseconds()); } else From 44b985a2bc9afd94fd206bb8699eabb69fae4e3f Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:26:32 +0100 Subject: [PATCH 164/332] Update install.md From 6b43d2e1971ac9303ca30692740c1469ddd39504 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 19 Mar 2024 13:27:26 +0100 Subject: [PATCH 165/332] Improve logging in build_download_helper --- tests/ci/build_download_helper.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 66734f58b23..b8469767876 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -120,12 +120,14 @@ def read_build_urls(build_name: str, reports_path: Union[Path, str]) -> List[str for root, _, files in os.walk(reports_path): for file in files: if file.endswith(f"_{build_name}.json"): - logging.info("Found build report json %s", file) + logging.info("Found build report json %s for %s", file, build_name) with open( os.path.join(root, file), "r", encoding="utf-8" ) as file_handler: build_report = json.load(file_handler) return build_report["build_urls"] # type: ignore + + logging.info("A build report is not found for %s", build_name) return [] @@ -197,7 +199,7 @@ def download_builds_filter( ): build_name = get_build_name_for_check(check_name) urls = read_build_urls(build_name, reports_path) - print(urls) + logging.info("The build report for %s contains the next URLs: %s", build_name, urls) if not urls: raise DownloadException("No build URLs found") @@ -223,6 +225,21 @@ def download_clickhouse_binary(check_name, reports_path, result_path): ) +def get_clickhouse_binary_url(check_name, reports_path): + build_name = get_build_name_for_check(check_name) + urls = read_build_urls(build_name, reports_path) + logging.info("The build report for %s contains the next URLs: %s", build_name, urls) + for url in urls: + check_url = url + if "?" in check_url: + check_url = check_url.split("?")[0] + + if check_url.endswith("clickhouse"): + return url + + return None + + def download_performance_build(check_name, reports_path, result_path): download_builds_filter( check_name, From 4efddb5ed47e6c7c323e9138b82e47cc71e7831c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 19 Mar 2024 14:01:31 +0100 Subject: [PATCH 166/332] Add type checking to build_download_helper --- tests/ci/build_download_helper.py | 37 +++++++++++++++++------- tests/ci/performance_comparison_check.py | 2 +- tests/ci/unit_tests_check.py | 2 +- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index b8469767876..0d89515d5d8 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -6,7 +6,7 @@ import os import sys import time from pathlib import Path -from typing import Any, Callable, List, Union +from typing import Any, Callable, List, Optional, Union # isort: off import requests @@ -185,18 +185,21 @@ def download_build_with_progress(url: str, path: Path) -> None: def download_builds( - result_path: str, build_urls: List[str], filter_fn: Callable[[str], bool] + result_path: Path, build_urls: List[str], filter_fn: Callable[[str], bool] ) -> None: for url in build_urls: if filter_fn(url): fname = os.path.basename(url.replace("%2B", "+").replace("%20", " ")) logging.info("Will download %s to %s", fname, result_path) - download_build_with_progress(url, Path(result_path) / fname) + download_build_with_progress(url, result_path / fname) def download_builds_filter( - check_name, reports_path, result_path, filter_fn=lambda _: True -): + check_name: str, + reports_path: Union[Path, str], + result_path: Path, + filter_fn: Callable[[str], bool] = lambda _: True, +) -> None: build_name = get_build_name_for_check(check_name) urls = read_build_urls(build_name, reports_path) logging.info("The build report for %s contains the next URLs: %s", build_name, urls) @@ -207,25 +210,33 @@ def download_builds_filter( download_builds(result_path, urls, filter_fn) -def download_all_deb_packages(check_name, reports_path, result_path): +def download_all_deb_packages( + check_name: str, reports_path: Union[Path, str], result_path: Path +) -> None: download_builds_filter( check_name, reports_path, result_path, lambda x: x.endswith("deb") ) -def download_unit_tests(check_name, reports_path, result_path): +def download_unit_tests( + check_name: str, reports_path: Union[Path, str], result_path: Path +) -> None: download_builds_filter( check_name, reports_path, result_path, lambda x: x.endswith("unit_tests_dbms") ) -def download_clickhouse_binary(check_name, reports_path, result_path): +def download_clickhouse_binary( + check_name: str, reports_path: Union[Path, str], result_path: Path +) -> None: download_builds_filter( check_name, reports_path, result_path, lambda x: x.endswith("clickhouse") ) -def get_clickhouse_binary_url(check_name, reports_path): +def get_clickhouse_binary_url( + check_name: str, reports_path: Union[Path, str] +) -> Optional[str]: build_name = get_build_name_for_check(check_name) urls = read_build_urls(build_name, reports_path) logging.info("The build report for %s contains the next URLs: %s", build_name, urls) @@ -240,7 +251,9 @@ def get_clickhouse_binary_url(check_name, reports_path): return None -def download_performance_build(check_name, reports_path, result_path): +def download_performance_build( + check_name: str, reports_path: Union[Path, str], result_path: Path +) -> None: download_builds_filter( check_name, reports_path, @@ -249,7 +262,9 @@ def download_performance_build(check_name, reports_path, result_path): ) -def download_fuzzers(check_name, reports_path, result_path): +def download_fuzzers( + check_name: str, reports_path: Union[Path, str], result_path: Path +) -> None: download_builds_filter( check_name, reports_path, diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index f0af15397c7..c238fbae603 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -155,7 +155,7 @@ def main(): } download_builds_filter( - check_name, REPORT_PATH, TEMP_PATH, lambda url: "performance.tar.zst" in url + check_name, REPORT_PATH, temp_path, lambda url: "performance.tar.zst" in url ) assert os.path.exists(f"{TEMP_PATH}/performance.tar.zst"), "Perf artifact not found" diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 2c2862d926a..b66a4312657 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -166,7 +166,7 @@ def main(): docker_image = pull_image(get_docker_image(IMAGE_NAME)) - download_unit_tests(check_name, REPORT_PATH, TEMP_PATH) + download_unit_tests(check_name, REPORT_PATH, temp_path) tests_binary = temp_path / "unit_tests_dbms" os.chmod(tests_binary, 0o777) From 939554ec0a62aa5a1b0395a72b95a17be8578d1b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Sat, 9 Mar 2024 07:52:23 +0000 Subject: [PATCH 167/332] contrib: update cppkafka to v0.4.1@9c5ea0e3 --- contrib/cppkafka | 2 +- src/Storages/Kafka/KafkaConsumer.cpp | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/contrib/cppkafka b/contrib/cppkafka index 5a119f689f8..9c5ea0e3324 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1 +Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d diff --git a/src/Storages/Kafka/KafkaConsumer.cpp b/src/Storages/Kafka/KafkaConsumer.cpp index 47167e19a38..7075dcb71ca 100644 --- a/src/Storages/Kafka/KafkaConsumer.cpp +++ b/src/Storages/Kafka/KafkaConsumer.cpp @@ -599,24 +599,14 @@ void KafkaConsumer::setExceptionInfo(const std::string & text, bool with_stacktr exceptions_buffer.push_back({enriched_text, static_cast(Poco::Timestamp().epochTime())}); } -/* - * Needed until - * https://github.com/mfontanini/cppkafka/pull/309 - * is merged, - * because consumer->get_member_id() contains a leak - */ std::string KafkaConsumer::getMemberId() const { if (!consumer) return ""; - char * memberid_ptr = rd_kafka_memberid(consumer->get_handle()); - std::string memberid_string = memberid_ptr; - rd_kafka_mem_free(nullptr, memberid_ptr); - return memberid_string; + return consumer->get_member_id(); } - KafkaConsumer::Stat KafkaConsumer::getStat() const { KafkaConsumer::Stat::Assignments assignments; From 4e2e5a015b162d63dd192dc1cdb0cb622a524cf1 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:29:50 +0100 Subject: [PATCH 168/332] Fixes --- src/Interpreters/InterpreterCreateQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index fb421ee0a5e..38918918c92 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1087,6 +1087,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { + // In case of an ON CLUSTER query, the database may not be present on the initiator node auto database = DatabaseCatalog::instance().tryGetDatabase(database_name); if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { @@ -1253,6 +1254,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) DatabasePtr database; bool need_add_to_database = !create.temporary; + // In case of an ON CLUSTER query, the database may not be present on the initiator node if (need_add_to_database) database = DatabaseCatalog::instance().tryGetDatabase(database_name); @@ -1273,7 +1275,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) "CREATE AS SELECT is not supported with Replicated databases. Use separate CREATE and INSERT queries"); } - if (need_add_to_database && database && database->shouldReplicateQuery(getContext(), query_ptr)) + if (database && database->shouldReplicateQuery(getContext(), query_ptr)) { chassert(!ddl_guard); auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); From 56e48c5d64c6e9ab3441908c04014f3e3e87de74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 16:56:46 +0100 Subject: [PATCH 169/332] Increase compiler memory for riscv --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c4e16eace2..d345705ae53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,8 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # set CPU time limit to 1000 seconds set (RLIMIT_CPU 1000) - # -fsanitize=memory and address are too heavy - if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) + # Sanitizers are too heavy. Also RISCV has some extra memory requirements + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE OR ARCH_RISCV64) set (RLIMIT_DATA 10000000000) # 10G endif() From 6987622737a6ecf8dc9d7d49a6cfd4867dc36a5b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 Mar 2024 15:57:19 +0000 Subject: [PATCH 170/332] Remove some tests from analyzer_tech_debt --- tests/analyzer_tech_debt.txt | 4 +--- .../0_stateless/01083_expressions_in_engine_arguments.sql | 2 +- .../0_stateless/01925_join_materialized_columns.reference | 3 +++ .../0_stateless/01925_join_materialized_columns.sql | 8 +++++--- tests/queries/0_stateless/02354_vector_search_queries.sql | 2 -- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 7cd73705e77..7dd6d059a71 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -1,13 +1,11 @@ 00223_shard_distributed_aggregation_memory_efficient 00725_memory_tracking 01062_pm_all_join_with_block_continuation -01083_expressions_in_engine_arguments 01155_rename_move_materialized_view 01584_distributed_buffer_cannot_find_column 01624_soft_constraints -01747_join_view_filter_dictionary 01925_join_materialized_columns -02354_annoy +02354_vector_search_queries # Check after constants refactoring 02901_parallel_replicas_rollup # Flaky. Please don't delete them without fixing them: diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index b162fdb21fd..7e4f77ccdb4 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -28,7 +28,7 @@ CREATE TABLE url (n UInt64, col String) ENGINE=URL ( replace ( - 'https://localhost:8443/?query=' || 'select n, _table from ' || currentDatabase() || '.merge format CSV', ' ', '+' + 'http://localhost:8123/?query=' || 'select n, _table from ' || currentDatabase() || '.merge format CSV', ' ', '+' ), CSV ); diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference index 5125b322aed..1d2ab582038 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.reference +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -10,6 +10,8 @@ - 2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 - 2020-01-01 12:00:00 fact1 t1_val1 2019-01-01 12:00:00 fact4 t2_val2 2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 @@ -32,3 +34,4 @@ fact2t1_val2 fact2t1_val2 2020-01-01 12:00:00 2020-01-01 12:00:00 2020-01-01 12:00:00 +2020-01-01 12:00:00 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql index 271c230c35b..abb6fda3bfb 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.sql +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -32,7 +32,8 @@ SELECT t1.dt, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.dt; SELECT '-'; SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time; SELECT '-'; -SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time; +SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings allow_experimental_analyzer=0; +SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings allow_experimental_analyzer=1; SELECT '-'; SELECT * FROM t1 JOIN t2 ON t1.dt1 = t2.dt2 ORDER BY t1.time, t1.dimension_1, t2.time, t2.dimension_2; SELECT '-'; @@ -51,5 +52,6 @@ SELECT t1.time as talias FROM t1 JOIN t2 ON talias = t2.time; SELECT t1.time as talias FROM t1 JOIN t2 ON talias = t2.time_alias; SELECT t2.time as talias FROM t1 JOIN t2 ON t1.time = talias; SELECT t2.time as talias FROM t1 JOIN t2 ON t1.time_alias = talias; -SELECT time as talias FROM t1 JOIN t2 ON t1.time = talias; -- { serverError AMBIGUOUS_COLUMN_NAME } -SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time; -- { serverError AMBIGUOUS_COLUMN_NAME } +SELECT time as talias FROM t1 JOIN t2 ON t1.time = talias; -- { serverError AMBIGUOUS_COLUMN_NAME, INVALID_JOIN_ON_EXPRESSION } +SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings allow_experimental_analyzer=0; -- { serverError AMBIGUOUS_COLUMN_NAME } +SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings allow_experimental_analyzer=1; diff --git a/tests/queries/0_stateless/02354_vector_search_queries.sql b/tests/queries/0_stateless/02354_vector_search_queries.sql index 2ccf869fdba..64051aa8544 100644 --- a/tests/queries/0_stateless/02354_vector_search_queries.sql +++ b/tests/queries/0_stateless/02354_vector_search_queries.sql @@ -8,8 +8,6 @@ SET allow_experimental_annoy_index = 1; SET allow_experimental_usearch_index = 1; -SET allow_experimental_analyzer = 0; - SELECT 'ARRAY, 10 rows, index_granularity = 8192, GRANULARITY = 1 million --> 1 granule, 1 indexed block'; DROP TABLE IF EXISTS tab_annoy; From b1d8521a5bff044e2fb53eb548c9810723a892e9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 19 Mar 2024 16:58:33 +0100 Subject: [PATCH 171/332] Update 01083_expressions_in_engine_arguments.sql --- .../0_stateless/01083_expressions_in_engine_arguments.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 7e4f77ccdb4..b162fdb21fd 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -28,7 +28,7 @@ CREATE TABLE url (n UInt64, col String) ENGINE=URL ( replace ( - 'http://localhost:8123/?query=' || 'select n, _table from ' || currentDatabase() || '.merge format CSV', ' ', '+' + 'https://localhost:8443/?query=' || 'select n, _table from ' || currentDatabase() || '.merge format CSV', ' ', '+' ), CSV ); From de855ca917798adea684d25bfb8d287303c30d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 17:04:29 +0100 Subject: [PATCH 172/332] Reduce header dependencies --- base/base/IPv4andIPv6.h | 6 +- programs/benchmark/Benchmark.cpp | 3 +- programs/format/Format.cpp | 3 +- programs/server/Server.cpp | 1 + src/Access/Common/QuotaDefs.cpp | 1 + src/Access/IAccessStorage.h | 2 + src/Access/User.cpp | 1 + .../AggregateFunctionRetention.cpp | 8 +-- .../AggregateFunctionSequenceMatch.cpp | 2 + src/Analyzer/Passes/QueryAnalysisPass.cpp | 1 + src/Backups/BackupEntriesCollector.cpp | 1 + src/Backups/BackupIO_Default.h | 1 + src/Backups/BackupIO_Disk.h | 2 + src/Backups/BackupIO_S3.h | 1 + src/Client/ClientBase.cpp | 9 +-- src/Client/LocalConnection.cpp | 5 +- src/Common/HTTPConnectionPool.cpp | 11 ++-- src/Common/Scheduler/IResourceManager.h | 3 + src/Common/Scheduler/ResouceLink.cpp | 25 ++++++++ src/Common/Scheduler/ResourceGuard.h | 3 +- src/Common/Scheduler/ResourceLink.h | 24 ++----- src/Common/SymbolIndex.cpp | 4 +- src/Common/ThreadStatus.h | 6 +- src/Common/formatIPv6.h | 7 +- ..._proxy_configuration_resolver_provider.cpp | 2 + src/Core/BaseSettings.h | 64 ------------------- src/Core/BaseSettingsProgramOptions.h | 60 +++++++++++++++++ src/Core/LogsLevel.h | 16 +++++ src/Core/SettingsEnums.h | 19 ++---- src/Daemon/SentryWriter.cpp | 1 + src/DataTypes/DataTypeDomainBool.cpp | 5 +- src/DataTypes/DataTypeFactory.h | 1 - src/DataTypes/DataTypeTuple.cpp | 2 +- src/DataTypes/DataTypeTuple.h | 2 +- src/DataTypes/IDataType.cpp | 2 +- src/DataTypes/IDataType.h | 9 ++- src/DataTypes/ObjectUtils.h | 1 + .../Serializations/SerializationInfo.h | 26 ++++---- .../SerializationInfoSettings.h | 14 ++++ .../gtest_DataType_deserializeAsText.cpp | 1 + src/Databases/DatabaseAtomic.cpp | 1 + src/Databases/DatabaseMemory.cpp | 1 + src/Databases/DatabaseOnDisk.cpp | 3 +- src/Databases/DatabaseReplicated.cpp | 1 + src/Databases/DatabasesCommon.cpp | 1 + src/Databases/IDatabase.cpp | 9 +-- .../ObjectStorages/DiskObjectStorage.cpp | 1 + src/Formats/FormatFactory.h | 1 + src/Formats/FormatSchemaInfo.h | 1 + src/Formats/ReadSchemaUtils.cpp | 1 + src/Formats/SchemaInferenceUtils.h | 6 ++ src/Functions/DateTimeTransforms.h | 1 + src/Functions/FunctionJoinGet.cpp | 7 +- .../FunctionsStringHashFixedString.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 3 +- src/Functions/array/arrayIntersect.cpp | 1 + src/Functions/dateDiff.cpp | 2 +- src/Functions/date_trunc.cpp | 3 +- src/Functions/hasColumnInTable.cpp | 1 + .../keyvaluepair/ArgumentExtractor.h | 1 + src/Functions/tupleConcat.cpp | 2 + src/IO/CachedInMemoryReadBufferFromFile.h | 3 +- src/IO/ReadHelpers.h | 9 ++- src/IO/S3/copyS3File.h | 1 + src/IO/examples/read_buffer_from_hdfs.cpp | 2 + src/Interpreters/ActionLocksManager.cpp | 1 + src/Interpreters/ActionsVisitor.cpp | 1 + src/Interpreters/ActionsVisitor.h | 5 +- src/Interpreters/AsynchronousInsertQueue.cpp | 1 + src/Interpreters/CollectJoinOnKeysVisitor.h | 7 +- src/Interpreters/Context.h | 15 ++++- src/Interpreters/DDLTask.cpp | 3 +- src/Interpreters/DatabaseCatalog.cpp | 40 ++++++++++++ src/Interpreters/DatabaseCatalog.h | 64 ------------------- src/Interpreters/IKeyValueEntity.h | 1 + .../InJoinSubqueriesPreprocessor.cpp | 1 + src/Interpreters/InterpreterCheckQuery.cpp | 1 + .../InterpreterCreateIndexQuery.cpp | 5 +- src/Interpreters/InterpreterDescribeQuery.cpp | 1 + .../InterpreterDropIndexQuery.cpp | 1 + src/Interpreters/InterpreterDropQuery.cpp | 1 + src/Interpreters/InterpreterExistsQuery.cpp | 1 + src/Interpreters/InterpreterInsertQuery.cpp | 1 + .../InterpreterKillQueryQuery.cpp | 1 + src/Interpreters/InterpreterOptimizeQuery.cpp | 1 + src/Interpreters/InterpreterRenameQuery.cpp | 1 + .../InterpreterShowCreateQuery.cpp | 1 + .../InterpreterShowFunctionsQuery.cpp | 6 +- src/Interpreters/InterpreterUndropQuery.cpp | 1 + src/Interpreters/InterpreterWatchQuery.cpp | 3 +- src/Interpreters/JoinedTables.cpp | 1 + ...writeSumFunctionWithSumAndCountVisitor.cpp | 2 + src/Interpreters/SystemLog.cpp | 1 + src/Interpreters/SystemLog.h | 3 +- src/Interpreters/TableNameHints.h | 39 +++++++++++ src/Interpreters/executeDDLQueryOnCluster.cpp | 1 + src/Interpreters/getTableExpressions.cpp | 1 + src/Interpreters/interpretSubquery.cpp | 5 +- src/Interpreters/loadMetadata.cpp | 5 +- src/Planner/PlannerJoinTree.cpp | 1 + .../Impl/JSONColumnsBlockInputFormatBase.h | 1 + src/Processors/Formats/Impl/Parquet/Write.h | 1 + .../CreateSetAndFilterOnTheFlyTransform.cpp | 2 + .../getSourceFromASTInsertQuery.cpp | 1 + src/Server/GRPCServer.cpp | 1 + src/Server/MySQLHandler.cpp | 5 +- src/Server/PrometheusMetricsWriter.cpp | 2 + .../Distributed/DistributedSettings.cpp | 7 +- src/Storages/ExecutableSettings.cpp | 6 +- src/Storages/ExecutableSettings.h | 1 + src/Storages/FileLog/StorageFileLog.cpp | 1 + .../extractZkPathFromCreateQuery.cpp | 7 +- src/Storages/RocksDB/StorageSystemRocksDB.cpp | 1 + src/Storages/S3Queue/StorageS3Queue.cpp | 3 +- src/Storages/StorageMaterializedView.cpp | 7 +- src/Storages/StorageNull.cpp | 3 +- src/Storages/System/StorageSystemClusters.cpp | 1 + src/Storages/System/StorageSystemColumns.cpp | 1 + .../System/StorageSystemDatabases.cpp | 1 + .../System/StorageSystemDistributionQueue.cpp | 1 + .../StorageSystemDroppedTablesParts.cpp | 1 + src/Storages/System/StorageSystemGraphite.cpp | 1 + .../System/StorageSystemMutations.cpp | 1 + .../System/StorageSystemPartsBase.cpp | 1 + src/Storages/System/StorageSystemTables.cpp | 1 + .../StorageSystemZooKeeperConnection.cpp | 2 + src/Storages/buildQueryTreeForShard.cpp | 1 + src/Storages/getStructureOfRemoteTable.cpp | 1 + src/TableFunctions/TableFunctionMerge.cpp | 1 + 129 files changed, 439 insertions(+), 266 deletions(-) create mode 100644 src/Common/Scheduler/ResouceLink.cpp create mode 100644 src/Core/BaseSettingsProgramOptions.h create mode 100644 src/Core/LogsLevel.h create mode 100644 src/DataTypes/Serializations/SerializationInfoSettings.h create mode 100644 src/Interpreters/TableNameHints.h diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h index e2f93b54124..9b1e518c161 100644 --- a/base/base/IPv4andIPv6.h +++ b/base/base/IPv4andIPv6.h @@ -1,8 +1,7 @@ #pragma once -#include #include -#include +#include #include namespace DB @@ -62,7 +61,8 @@ namespace std { size_t operator()(const DB::IPv6 & x) const { - return std::hash{}(std::string_view(reinterpret_cast(&x.toUnderType()), IPV6_BINARY_LENGTH)); + return std::hash{}( + std::string_view(reinterpret_cast(&x.toUnderType()), sizeof(DB::IPv6::UnderlyingType))); } }; diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 45dadfef774..eecc352d073 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -34,6 +34,7 @@ #include #include #include +#include /** A tool for evaluating ClickHouse performance. @@ -623,7 +624,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ; Settings settings; - settings.addProgramOptions(desc); + addProgramOptions(settings, desc); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 50f801f2560..fc73eda6815 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -102,7 +103,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { std::string_view name = field.getName(); if (name == "max_parser_depth" || name == "max_query_size") - cmd_settings.addProgramOption(desc, name, field); + addProgramOption(cmd_settings, desc, name, field); } boost::program_options::variables_map options; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b67a4eccd15..59dadd14b20 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Access/Common/QuotaDefs.cpp b/src/Access/Common/QuotaDefs.cpp index 04c16a562d2..2a22b80f65d 100644 --- a/src/Access/Common/QuotaDefs.cpp +++ b/src/Access/Common/QuotaDefs.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index ad78bf92e02..b00c8bac849 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -13,6 +13,8 @@ #include #include +#include + namespace Poco { class Logger; } namespace Poco::Net { class IPAddress; } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 91477e5f3aa..39930c9cf76 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index a004f3527a2..5eaa1a7a39c 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -1,16 +1,16 @@ #include +#include #include - -#include #include #include #include #include #include #include -#include +#include -#include +#include +#include namespace DB diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp index ff9259e3aac..bc3d4da5e39 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp @@ -10,6 +10,8 @@ #include #include #include +#include + #include #include diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1da5d86edf3..d8a196ee3d4 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 5c0b80aa894..c71ce195388 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Backups/BackupIO_Default.h b/src/Backups/BackupIO_Default.h index 639293f22d9..4dba0324703 100644 --- a/src/Backups/BackupIO_Default.h +++ b/src/Backups/BackupIO_Default.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 575ec3f5707..3d3253877bd 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include + #include diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 8ab8e1fb566..57108d122ea 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -4,6 +4,7 @@ #if USE_AWS_S3 #include +#include #include #include #include diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index d561a64895b..582dc2c9e6b 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -2714,9 +2715,9 @@ private: void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { if (allow_repeated_settings) - cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); + addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value()); else - cmd_settings.addProgramOptions(options_description.main_description.value()); + addProgramOptions(cmd_settings, options_description.main_description.value()); if (allow_merge_tree_settings) { @@ -2737,9 +2738,9 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, return; if (allow_repeated_settings) - cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, name, setting); + addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting); else - cmd_merge_tree_settings.addProgramOption(main_options, name, setting); + addProgramOption(cmd_merge_tree_settings, main_options, name, setting); }; const auto & setting_name = setting.getName(); diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index dbb115f44ef..afcaa4d6098 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -1,13 +1,14 @@ #include "LocalConnection.h" +#include +#include #include #include #include -#include #include +#include #include #include #include -#include namespace DB diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index a21438a11a2..f729b8ea8d0 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -10,14 +10,15 @@ #include #include -#include -#include -#include #include +#include +#include +#include +#include +#include #include -#include -#include +#include #include "config.h" diff --git a/src/Common/Scheduler/IResourceManager.h b/src/Common/Scheduler/IResourceManager.h index 0af74e00960..8a7077ac3d5 100644 --- a/src/Common/Scheduler/IResourceManager.h +++ b/src/Common/Scheduler/IResourceManager.h @@ -12,6 +12,9 @@ namespace DB { +class ISchedulerNode; +using SchedulerNodePtr = std::shared_ptr; + /* * Instance of derived class holds everything required for resource consumption, * including resources currently registered at `SchedulerRoot`. This is required to avoid diff --git a/src/Common/Scheduler/ResouceLink.cpp b/src/Common/Scheduler/ResouceLink.cpp new file mode 100644 index 00000000000..2da5dba62dc --- /dev/null +++ b/src/Common/Scheduler/ResouceLink.cpp @@ -0,0 +1,25 @@ +#include +#include +#include + +namespace DB +{ +void ResourceLink::adjust(ResourceCost estimated_cost, ResourceCost real_cost) const +{ + if (queue) + queue->adjustBudget(estimated_cost, real_cost); +} + +void ResourceLink::consumed(ResourceCost cost) const +{ + if (queue) + queue->consumeBudget(cost); +} + +void ResourceLink::accumulate(DB::ResourceCost cost) const +{ + if (queue) + queue->accumulateBudget(cost); +} +} + diff --git a/src/Common/Scheduler/ResourceGuard.h b/src/Common/Scheduler/ResourceGuard.h index 50f665a384b..3c29f588fba 100644 --- a/src/Common/Scheduler/ResourceGuard.h +++ b/src/Common/Scheduler/ResourceGuard.h @@ -2,9 +2,10 @@ #include +#include +#include #include #include -#include #include #include diff --git a/src/Common/Scheduler/ResourceLink.h b/src/Common/Scheduler/ResourceLink.h index 108f51ac399..450d9bc1efa 100644 --- a/src/Common/Scheduler/ResourceLink.h +++ b/src/Common/Scheduler/ResourceLink.h @@ -2,12 +2,10 @@ #include -#include -#include - - namespace DB { +class ISchedulerQueue; +using ResourceCost = Int64; /* * Everything required for resource consumption. Connection to a specific resource queue. @@ -17,23 +15,11 @@ struct ResourceLink ISchedulerQueue * queue = nullptr; bool operator==(const ResourceLink &) const = default; - void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const - { - if (queue) - queue->adjustBudget(estimated_cost, real_cost); - } + void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const; - void consumed(ResourceCost cost) const - { - if (queue) - queue->consumeBudget(cost); - } + void consumed(ResourceCost cost) const; - void accumulate(ResourceCost cost) const - { - if (queue) - queue->accumulateBudget(cost); - } + void accumulate(ResourceCost cost) const; }; } diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 8dde617fc74..a2d58fc7e8c 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -11,8 +13,6 @@ #include -#include - /** ELF object can contain three different places with symbol names and addresses: diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 77386af6c2b..48b52f8aa6e 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -1,12 +1,12 @@ #pragma once -#include -#include +#include #include +#include +#include #include #include #include -#include #include diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h index fa7a9592c48..3451eda6b3c 100644 --- a/src/Common/formatIPv6.h +++ b/src/Common/formatIPv6.h @@ -1,13 +1,12 @@ #pragma once -#include -#include #include +#include #include #include -#include -#include #include +#include +#include #include constexpr size_t IPV4_BINARY_LENGTH = 4; diff --git a/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp b/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp index 9a7447b02e4..d5d6f86f661 100644 --- a/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp +++ b/src/Common/tests/gtest_proxy_configuration_resolver_provider.cpp @@ -4,6 +4,8 @@ #include #include +#include + using ConfigurationPtr = Poco::AutoPtr; class ProxyConfigurationResolverProviderTests : public ::testing::Test diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 6f3245c83e8..7191038a4ce 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -7,7 +7,6 @@ #include #include #include -#include namespace boost::program_options @@ -129,18 +128,6 @@ public: std::conditional_t custom_setting; }; - /// Adds program options to set the settings from a command line. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptions(boost::program_options::options_description & options); - - /// Adds program options as to set the settings from a command line. - /// Allows to set one setting multiple times, the last value will be used. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptionsAsMultitokens(boost::program_options::options_description & options); - - void addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field); - void addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field); - enum SkipFlags { SKIP_NONE = 0, @@ -561,57 +548,6 @@ String BaseSettings::toString() const return out.str(); } -template -void BaseSettings::addProgramOptions(boost::program_options::options_description & options) -{ - const auto & settings_to_aliases = TTraits::settingsToAliases(); - for (const auto & field : all()) - { - std::string_view name = field.getName(); - addProgramOption(options, name, field); - - if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) - { - for (const auto alias : it->second) - addProgramOption(options, alias, field); - } - } -} - -template -void BaseSettings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options) -{ - const auto & settings_to_aliases = TTraits::settingsToAliases(); - for (const auto & field : all()) - { - std::string_view name = field.getName(); - addProgramOptionAsMultitoken(options, name, field); - - if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) - { - for (const auto alias : it->second) - addProgramOptionAsMultitoken(options, alias, field); - } - } -} - - -template -void BaseSettings::addProgramOption(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field) -{ - auto on_program_option = boost::function1([this, name](const std::string & value) { set(name, value); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); -} - -template -void BaseSettings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, std::string_view name, const SettingFieldRef & field) -{ - auto on_program_option = boost::function1([this, name](const Strings & values) { set(name, values.back()); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); -} - template bool operator==(const BaseSettings & left, const BaseSettings & right) { diff --git a/src/Core/BaseSettingsProgramOptions.h b/src/Core/BaseSettingsProgramOptions.h new file mode 100644 index 00000000000..6c8166fc119 --- /dev/null +++ b/src/Core/BaseSettingsProgramOptions.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include + +namespace DB +{ + +template +void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) +{ + auto on_program_option = boost::function1([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); +} + +template +void addProgramOptionsAsMultitokens(T &cmd_settings, boost::program_options::options_description & options) +{ + const auto & settings_to_aliases = T::Traits::settingsToAliases(); + for (const auto & field : cmd_settings.all()) + { + std::string_view name = field.getName(); + addProgramOptionAsMultitoken(cmd_settings, options, name, field); + + if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) + for (const auto alias : it->second) + addProgramOptionAsMultitoken(cmd_settings, options, alias, field); + } +} + +/// Adds program options to set the settings from a command line. +/// (Don't forget to call notify() on the `variables_map` after parsing it!) +template +void addProgramOption(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) +{ + auto on_program_option = boost::function1([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); +} + +template +void addProgramOptions(T &cmd_settings, boost::program_options::options_description & options) +{ + const auto & settings_to_aliases = T::Traits::settingsToAliases(); + for (const auto & field : cmd_settings.all()) + { + std::string_view name = field.getName(); + addProgramOption(cmd_settings, options, name, field); + + if (auto it = settings_to_aliases.find(name); it != settings_to_aliases.end()) + for (const auto alias : it->second) + addProgramOption(cmd_settings, options, alias, field); + } +} + + +} diff --git a/src/Core/LogsLevel.h b/src/Core/LogsLevel.h new file mode 100644 index 00000000000..95721820d9c --- /dev/null +++ b/src/Core/LogsLevel.h @@ -0,0 +1,16 @@ +#pragma once + +namespace DB +{ +enum class LogsLevel +{ + none = 0, /// Disable + fatal, + error, + warning, + information, + debug, + trace, + test, +}; +} diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 22fcf0389d8..421712ed868 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -1,12 +1,13 @@ #pragma once -#include #include -#include +#include +#include #include #include -#include #include +#include +#include namespace DB @@ -90,18 +91,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(IntervalOutputFormat, FormatSettings::IntervalO DECLARE_SETTING_ENUM_WITH_RENAME(ParquetVersion, FormatSettings::ParquetVersion) -enum class LogsLevel -{ - none = 0, /// Disable - fatal, - error, - warning, - information, - debug, - trace, - test, -}; - DECLARE_SETTING_ENUM(LogsLevel) diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index 192e9952b9a..0fa8b99a6f7 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "config.h" #include diff --git a/src/DataTypes/DataTypeDomainBool.cpp b/src/DataTypes/DataTypeDomainBool.cpp index 245c5495299..3d19b6262d8 100644 --- a/src/DataTypes/DataTypeDomainBool.cpp +++ b/src/DataTypes/DataTypeDomainBool.cpp @@ -1,6 +1,7 @@ -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/DataTypes/DataTypeFactory.h b/src/DataTypes/DataTypeFactory.h index a2aeb6f3646..4727cb3ae5c 100644 --- a/src/DataTypes/DataTypeFactory.h +++ b/src/DataTypes/DataTypeFactory.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index eb218d8efb7..5bbd79160d4 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -346,7 +346,7 @@ SerializationPtr DataTypeTuple::getSerialization(const SerializationInfo & info) return std::make_shared(std::move(serializations), have_explicit_names); } -MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfo::Settings & settings) const +MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfoSettings & settings) const { MutableSerializationInfos infos; infos.reserve(elems.size()); diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h index 4e5a0c1b33c..15561fe4286 100644 --- a/src/DataTypes/DataTypeTuple.h +++ b/src/DataTypes/DataTypeTuple.h @@ -58,7 +58,7 @@ public: SerializationPtr doGetDefaultSerialization() const override; SerializationPtr getSerialization(const SerializationInfo & info) const override; - MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const override; + MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const override; SerializationInfoPtr getSerializationInfo(const IColumn & column) const override; const DataTypePtr & getElement(size_t i) const { return elems[i]; } diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index 40915418aea..344b81be960 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -202,7 +202,7 @@ void IDataType::setCustomization(DataTypeCustomDescPtr custom_desc_) const custom_serialization = std::move(custom_desc_->serialization); } -MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfo::Settings & settings) const +MutableSerializationInfoPtr IDataType::createSerializationInfo(const SerializationInfoSettings & settings) const { return std::make_shared(ISerialization::Kind::DEFAULT, settings); } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 4403e3d9bd4..eaf798a3017 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -7,8 +7,6 @@ #include #include #include -#include - namespace DB { @@ -38,6 +36,11 @@ struct DataTypeWithConstInfo using DataTypesWithConstInfo = std::vector; +class SerializationInfo; +using SerializationInfoPtr = std::shared_ptr; +using MutableSerializationInfoPtr = std::shared_ptr; +struct SerializationInfoSettings; + /** Properties of data type. * * Contains methods for getting serialization instances. @@ -117,7 +120,7 @@ public: Names getSubcolumnNames() const; - virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfo::Settings & settings) const; + virtual MutableSerializationInfoPtr createSerializationInfo(const SerializationInfoSettings & settings) const; virtual SerializationInfoPtr getSerializationInfo(const IColumn & column) const; /// TODO: support more types. diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 2bfcaae09ca..9897f7d20f9 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -13,6 +13,7 @@ namespace DB struct StorageSnapshot; using StorageSnapshotPtr = std::shared_ptr; +class ColumnsDescription; /// Returns number of dimensions in Array type. 0 if type is not array. size_t getNumberOfDimensions(const IDataType & type); diff --git a/src/DataTypes/Serializations/SerializationInfo.h b/src/DataTypes/Serializations/SerializationInfo.h index 3d8f4f1d00c..5a900a5521c 100644 --- a/src/DataTypes/Serializations/SerializationInfo.h +++ b/src/DataTypes/Serializations/SerializationInfo.h @@ -2,6 +2,8 @@ #include #include +#include + #include @@ -28,6 +30,8 @@ constexpr auto SERIALIZATION_INFO_VERSION = 0; class SerializationInfo { public: + using Settings = SerializationInfoSettings; + struct Data { size_t num_rows = 0; @@ -38,16 +42,8 @@ public: void addDefaults(size_t length); }; - struct Settings - { - const double ratio_of_defaults_for_sparse = 1.0; - const bool choose_kind = false; - - bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; } - }; - - SerializationInfo(ISerialization::Kind kind_, const Settings & settings_); - SerializationInfo(ISerialization::Kind kind_, const Settings & settings_, const Data & data_); + SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_); + SerializationInfo(ISerialization::Kind kind_, const SerializationInfoSettings & settings_, const Data & data_); virtual ~SerializationInfo() = default; @@ -64,7 +60,7 @@ public: virtual std::shared_ptr createWithType( const IDataType & old_type, const IDataType & new_type, - const Settings & new_settings) const; + const SerializationInfoSettings & new_settings) const; virtual void serialializeKindBinary(WriteBuffer & out) const; virtual void deserializeFromKindsBinary(ReadBuffer & in); @@ -73,14 +69,14 @@ public: virtual void fromJSON(const Poco::JSON::Object & object); void setKind(ISerialization::Kind kind_) { kind = kind_; } - const Settings & getSettings() const { return settings; } + const SerializationInfoSettings & getSettings() const { return settings; } const Data & getData() const { return data; } ISerialization::Kind getKind() const { return kind; } - static ISerialization::Kind chooseKind(const Data & data, const Settings & settings); + static ISerialization::Kind chooseKind(const Data & data, const SerializationInfoSettings & settings); protected: - const Settings settings; + const SerializationInfoSettings settings; ISerialization::Kind kind; Data data; @@ -96,7 +92,7 @@ using MutableSerializationInfos = std::vector; class SerializationInfoByName : public std::map { public: - using Settings = SerializationInfo::Settings; + using Settings = SerializationInfoSettings; SerializationInfoByName() = default; SerializationInfoByName(const NamesAndTypesList & columns, const Settings & settings); diff --git a/src/DataTypes/Serializations/SerializationInfoSettings.h b/src/DataTypes/Serializations/SerializationInfoSettings.h new file mode 100644 index 00000000000..26f2c344bea --- /dev/null +++ b/src/DataTypes/Serializations/SerializationInfoSettings.h @@ -0,0 +1,14 @@ +#pragma once + +namespace DB +{ + +struct SerializationInfoSettings +{ + const double ratio_of_defaults_for_sparse = 1.0; + const bool choose_kind = false; + + bool isAlwaysDefault() const { return ratio_of_defaults_for_sparse >= 1.0; } +}; + +} diff --git a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp index 0373e55a62d..bf5337c89da 100644 --- a/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp +++ b/src/DataTypes/tests/gtest_DataType_deserializeAsText.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 0bf7c8af4b4..6f3f1151153 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 794eebbc399..4ff7b3c7f2b 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index dcfc1916450..896baa561b4 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -5,8 +5,9 @@ #include #include #include -#include #include +#include +#include #include #include #include diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 3b6a712510d..59b3e52e139 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index f8d6ad69ba8..d40e5f98aaa 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index ae8fc58bf89..95d671d1960 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -1,11 +1,12 @@ #include #include -#include -#include -#include #include -#include +#include +#include +#include #include +#include +#include namespace CurrentMetrics diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 696a0ef7416..9d20427e0d5 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 46c1b8ddcdd..93f67e83831 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -23,6 +23,7 @@ namespace DB class Block; struct Settings; struct FormatFactorySettings; +struct ReadSettings; class ReadBuffer; class WriteBuffer; diff --git a/src/Formats/FormatSchemaInfo.h b/src/Formats/FormatSchemaInfo.h index e8758c3f761..0dd28699896 100644 --- a/src/Formats/FormatSchemaInfo.h +++ b/src/Formats/FormatSchemaInfo.h @@ -8,6 +8,7 @@ namespace DB { class Context; +class Block; /// Extracts information about where the format schema file is from passed context and keep it. class FormatSchemaInfo diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index b05b768899b..735b536986d 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Formats/SchemaInferenceUtils.h b/src/Formats/SchemaInferenceUtils.h index b492d9b22b6..bcf3d194825 100644 --- a/src/Formats/SchemaInferenceUtils.h +++ b/src/Formats/SchemaInferenceUtils.h @@ -3,9 +3,15 @@ #include #include +#include + namespace DB { +class Block; +class NamesAndTypesList; +using NamesAndTypesLists = std::vector; + /// Struct with some additional information about inferred types for JSON formats. struct JSONInferenceInfo { diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index f5072de24f3..558c309007c 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 5602c88c60e..085c4db3f57 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -1,12 +1,13 @@ #include +#include #include #include -#include -#include #include +#include +#include +#include #include #include -#include namespace DB diff --git a/src/Functions/FunctionsStringHashFixedString.cpp b/src/Functions/FunctionsStringHashFixedString.cpp index bbb02de6c9f..f4160badd37 100644 --- a/src/Functions/FunctionsStringHashFixedString.cpp +++ b/src/Functions/FunctionsStringHashFixedString.cpp @@ -274,7 +274,7 @@ public: const typename ColumnIPv6::Container & data = col_from_ip->getData(); const auto size = col_from_ip->size(); auto & chars_to = col_to->getChars(); - const auto length = IPV6_BINARY_LENGTH; + const auto length = sizeof(IPv6::UnderlyingType); chars_to.resize(size * Impl::length); for (size_t i = 0; i < size; ++i) { diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 7ff35e599be..a5554171aaa 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -1,9 +1,10 @@ #include -#include #include #include #include #include +#include +#include namespace DB diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index ffc7480219e..209441eb301 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index f75e6eb4fc8..8f318d1bc55 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -6,7 +6,7 @@ #include #include #include - +#include #include #include #include diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index f64848e9185..8493df17a2f 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -1,9 +1,10 @@ #include -#include #include +#include #include #include #include +#include #include #include diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 66ed515e490..48783a672e2 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include diff --git a/src/Functions/keyvaluepair/ArgumentExtractor.h b/src/Functions/keyvaluepair/ArgumentExtractor.h index e6538584d01..6ff8aa36f13 100644 --- a/src/Functions/keyvaluepair/ArgumentExtractor.h +++ b/src/Functions/keyvaluepair/ArgumentExtractor.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace DB diff --git a/src/Functions/tupleConcat.cpp b/src/Functions/tupleConcat.cpp index 0556f4181e6..c48e4d61463 100644 --- a/src/Functions/tupleConcat.cpp +++ b/src/Functions/tupleConcat.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes diff --git a/src/IO/CachedInMemoryReadBufferFromFile.h b/src/IO/CachedInMemoryReadBufferFromFile.h index 300c2e82386..a0d07486359 100644 --- a/src/IO/CachedInMemoryReadBufferFromFile.h +++ b/src/IO/CachedInMemoryReadBufferFromFile.h @@ -1,7 +1,8 @@ #pragma once -#include #include +#include +#include namespace DB { diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 49530f4787a..ca568c469b4 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -154,9 +154,12 @@ inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) size_t size = 0; readVarUInt(size, buf); - if (size != IPV6_BINARY_LENGTH) - throw Exception(ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, - "Size of the string {} doesn't match size of binary IPv6 {}", size, IPV6_BINARY_LENGTH); + if (size != sizeof(IPv6::UnderlyingType)) + throw Exception( + ErrorCodes::SIZE_OF_FIXED_STRING_DOESNT_MATCH, + "Size of the string {} doesn't match size of binary IPv6 {}", + size, + sizeof(IPv6::UnderlyingType)); buf.readStrict(reinterpret_cast(&ip.toUnderType()), size); } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 093d26ba7bb..5eb6f702473 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -14,6 +14,7 @@ namespace DB { +struct ReadSettings; class SeekableReadBuffer; using CreateReadBuffer = std::function()>; diff --git a/src/IO/examples/read_buffer_from_hdfs.cpp b/src/IO/examples/read_buffer_from_hdfs.cpp index 977dd2ae227..c499542fedb 100644 --- a/src/IO/examples/read_buffer_from_hdfs.cpp +++ b/src/IO/examples/read_buffer_from_hdfs.cpp @@ -6,6 +6,8 @@ #include #include +#include + using namespace DB; int main() diff --git a/src/Interpreters/ActionLocksManager.cpp b/src/Interpreters/ActionLocksManager.cpp index 65f13ebd66c..28803a94c80 100644 --- a/src/Interpreters/ActionLocksManager.cpp +++ b/src/Interpreters/ActionLocksManager.cpp @@ -1,5 +1,6 @@ #include "ActionLocksManager.h" #include +#include #include #include diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f9a58c8c3ca..16e2449206d 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 1e2ebaf6d87..643bf50ecd4 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -1,13 +1,14 @@ #pragma once #include +#include +#include #include #include #include #include #include -#include -#include +#include namespace DB { diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 9234d052d97..1c6039bbbf7 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.h b/src/Interpreters/CollectJoinOnKeysVisitor.h index 194ec01bcd5..ff10db3beb5 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.h +++ b/src/Interpreters/CollectJoinOnKeysVisitor.h @@ -1,11 +1,12 @@ #pragma once +#include #include +#include +#include +#include #include #include -#include -#include -#include namespace DB diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index dbd9bbd1ff8..c2d6febc733 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -19,9 +19,8 @@ #include #include #include -#include +#include #include -#include #include #include #include @@ -149,6 +148,18 @@ template class MergeTreeBackgroundExecutor; class AsyncLoader; +struct TemporaryTableHolder; +using TemporaryTablesMapping = std::map>; + +class LoadTask; +using LoadTaskPtr = std::shared_ptr; +using LoadTaskPtrs = std::vector; + +class IClassifier; +using ClassifierPtr = std::shared_ptr; +class IResourceManager; +using ResourceManagerPtr = std::shared_ptr; + /// Scheduling policy can be changed using `background_merges_mutations_scheduling_policy` config option. /// By default concurrent merges are scheduled using "round_robin" to ensure fair and starvation-free operation. /// Previously in heavily overloaded shards big merges could possibly be starved by smaller diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index e10f3ecfbc9..a37b4db029a 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -14,7 +16,6 @@ #include #include #include -#include namespace DB diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index c2f2003aabd..ec6c8b5924f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -1705,4 +1706,43 @@ DDLGuard::~DDLGuard() releaseTableLock(); } +std::pair TableNameHints::getHintForTable(const String & table_name) const +{ + auto results = this->getHints(table_name, getAllRegisteredNames()); + if (results.empty()) + return getExtendedHintForTable(table_name); + return std::make_pair(database->getDatabaseName(), results[0]); +} + +std::pair TableNameHints::getExtendedHintForTable(const String & table_name) const +{ + /// load all available databases from the DatabaseCatalog instance + auto & database_catalog = DatabaseCatalog::instance(); + auto all_databases = database_catalog.getDatabases(); + + for (const auto & [db_name, db] : all_databases) + { + /// this case should be covered already by getHintForTable + if (db_name == database->getDatabaseName()) + continue; + + TableNameHints hints(db, context); + auto results = hints.getHints(table_name); + + /// if the results are not empty, return the first instance of the table_name + /// and the corresponding database_name that was found. + if (!results.empty()) + return std::make_pair(db_name, results[0]); + } + return {}; +} + +Names TableNameHints::getAllRegisteredNames() const +{ + Names result; + if (database) + for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) + result.emplace_back(table_it->name()); + return result; +} } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 6995fc51941..30b49cf1fed 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -1,13 +1,11 @@ #pragma once #include -#include #include #include #include #include #include -#include #include #include @@ -365,68 +363,6 @@ private: static constexpr time_t DBMS_DEFAULT_DISK_RELOAD_PERIOD_SEC = 5; }; -class TableNameHints : public IHints<> -{ -public: - TableNameHints(ConstDatabasePtr database_, ContextPtr context_) - : context(context_), - database(database_) - { - } - - /// getHintForTable tries to get a hint for the provided table_name in the provided - /// database. If the results are empty, it goes for extended hints for the table - /// with getExtendedHintForTable which looks for the table name in every database that's - /// available in the database catalog. It finally returns a single hint which is the database - /// name and table_name pair which is similar to the table_name provided. Perhaps something to - /// consider is should we return more than one pair of hint? - std::pair getHintForTable(const String & table_name) const - { - auto results = this->getHints(table_name, getAllRegisteredNames()); - if (results.empty()) - return getExtendedHintForTable(table_name); - return std::make_pair(database->getDatabaseName(), results[0]); - } - - /// getExtendedHintsForTable tries to get hint for the given table_name across all - /// the databases that are available in the database catalog. - std::pair getExtendedHintForTable(const String & table_name) const - { - /// load all available databases from the DatabaseCatalog instance - auto & database_catalog = DatabaseCatalog::instance(); - auto all_databases = database_catalog.getDatabases(); - - for (const auto & [db_name, db] : all_databases) - { - /// this case should be covered already by getHintForTable - if (db_name == database->getDatabaseName()) - continue; - - TableNameHints hints(db, context); - auto results = hints.getHints(table_name); - - /// if the results are not empty, return the first instance of the table_name - /// and the corresponding database_name that was found. - if (!results.empty()) - return std::make_pair(db_name, results[0]); - } - return {}; - } - - Names getAllRegisteredNames() const override - { - Names result; - if (database) - for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) - result.emplace_back(table_it->name()); - return result; - } - -private: - ContextPtr context; - ConstDatabasePtr database; -}; - /// This class is useful when creating a table or database. /// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog. diff --git a/src/Interpreters/IKeyValueEntity.h b/src/Interpreters/IKeyValueEntity.h index d1ceda57f0e..856ce28bae7 100644 --- a/src/Interpreters/IKeyValueEntity.h +++ b/src/Interpreters/IKeyValueEntity.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index ec4241a2740..3b3ef928b42 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 98a281bd5ad..ae8cef3f102 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index cd2f996c74c..aed4b0587b4 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -1,10 +1,11 @@ -#include -#include #include #include #include +#include #include +#include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 04d44e34fff..87b0eb89302 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 025677eeb91..f052aa201f1 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 72aa4cc63e3..a7040709034 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterExistsQuery.cpp b/src/Interpreters/InterpreterExistsQuery.cpp index e4176a44533..14ca9eaed0a 100644 --- a/src/Interpreters/InterpreterExistsQuery.cpp +++ b/src/Interpreters/InterpreterExistsQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 3e8bb268fe7..fc58f7b5098 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 26dae6a1df3..6d6b1085ffb 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 1e2eaa50ab1..907a01b0432 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index 06b6ebc9cbb..b63672f1757 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 9edac1fd8e1..0fca7b64d5a 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterShowFunctionsQuery.cpp b/src/Interpreters/InterpreterShowFunctionsQuery.cpp index 829670d7929..96a730f6527 100644 --- a/src/Interpreters/InterpreterShowFunctionsQuery.cpp +++ b/src/Interpreters/InterpreterShowFunctionsQuery.cpp @@ -1,8 +1,8 @@ -#include -#include - +#include #include #include +#include +#include #include #include diff --git a/src/Interpreters/InterpreterUndropQuery.cpp b/src/Interpreters/InterpreterUndropQuery.cpp index f628a656947..920df3d6aed 100644 --- a/src/Interpreters/InterpreterUndropQuery.cpp +++ b/src/Interpreters/InterpreterUndropQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp index 2b68c5d7a10..4937d8660e0 100644 --- a/src/Interpreters/InterpreterWatchQuery.cpp +++ b/src/Interpreters/InterpreterWatchQuery.cpp @@ -12,9 +12,10 @@ limitations under the License. */ #include #include #include +#include +#include #include #include -#include #include #include #include diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 49693332280..5b549a19083 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include diff --git a/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp b/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp index 2f5e597bdab..8e60603cca9 100644 --- a/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp +++ b/src/Interpreters/RewriteSumFunctionWithSumAndCountVisitor.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace DB { diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index efb2559ce42..db73fe038c0 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index c296b91e24a..e5b79585701 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -1,8 +1,9 @@ #pragma once +#include #include -#include +#include namespace DB { diff --git a/src/Interpreters/TableNameHints.h b/src/Interpreters/TableNameHints.h new file mode 100644 index 00000000000..d3d8d10c50c --- /dev/null +++ b/src/Interpreters/TableNameHints.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +#include + +namespace DB +{ + +class IDatabase; +using ConstDatabasePtr = std::shared_ptr; + +class TableNameHints : public IHints<> +{ +public: + TableNameHints(ConstDatabasePtr database_, ContextPtr context_) : context(context_), database(database_) { } + + /// getHintForTable tries to get a hint for the provided table_name in the provided + /// database. If the results are empty, it goes for extended hints for the table + /// with getExtendedHintForTable which looks for the table name in every database that's + /// available in the database catalog. It finally returns a single hint which is the database + /// name and table_name pair which is similar to the table_name provided. Perhaps something to + /// consider is should we return more than one pair of hint? + std::pair getHintForTable(const String & table_name) const; + + /// getExtendedHintsForTable tries to get hint for the given table_name across all + /// the databases that are available in the database catalog. + std::pair getExtendedHintForTable(const String & table_name) const; + + Names getAllRegisteredNames() const override; + +private: + ContextPtr context; + ConstDatabasePtr database; +}; + +} diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index df8236c11f4..fbcb57b6125 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 2853be4c05e..cd92d9ecbf3 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index 5f00be07fa5..6b42345f1d6 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -10,9 +10,10 @@ #include #include -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 226472175b3..9c3922b8bda 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -6,11 +6,12 @@ #include #include +#include +#include #include #include -#include -#include #include +#include #include #include diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8ca8f0f258b..a8d455cf9ee 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -51,6 +51,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h index 6d2532b350a..b1fd86bb806 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index 24733ac276b..f162984fd5e 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp b/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp index 83a75318d61..c38e9e97ed5 100644 --- a/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp +++ b/src/Processors/Transforms/CreateSetAndFilterOnTheFlyTransform.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace DB { diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index 8a13973b970..a2a42f27c3f 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index dfae812c620..72135157117 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 9efcebfc72d..ad5cae01540 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,10 +23,10 @@ #include #include #include -#include -#include #include +#include #include +#include #if USE_SSL # include diff --git a/src/Server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp index d0fdcd61493..85eafbe4808 100644 --- a/src/Server/PrometheusMetricsWriter.cpp +++ b/src/Server/PrometheusMetricsWriter.cpp @@ -1,7 +1,9 @@ #include "PrometheusMetricsWriter.h" #include +#include #include + #include #include "config.h" diff --git a/src/Storages/Distributed/DistributedSettings.cpp b/src/Storages/Distributed/DistributedSettings.cpp index e07b8da34af..1f6aa6c72fa 100644 --- a/src/Storages/Distributed/DistributedSettings.cpp +++ b/src/Storages/Distributed/DistributedSettings.cpp @@ -1,9 +1,12 @@ -#include +#include #include -#include #include +#include +#include #include +#include + namespace DB { diff --git a/src/Storages/ExecutableSettings.cpp b/src/Storages/ExecutableSettings.cpp index dc462350a06..d00e4098181 100644 --- a/src/Storages/ExecutableSettings.cpp +++ b/src/Storages/ExecutableSettings.cpp @@ -1,10 +1,10 @@ #include "ExecutableSettings.h" -#include - +#include #include -#include #include +#include +#include namespace DB { diff --git a/src/Storages/ExecutableSettings.h b/src/Storages/ExecutableSettings.h index 10dbae8ac9f..95627f08d16 100644 --- a/src/Storages/ExecutableSettings.h +++ b/src/Storages/ExecutableSettings.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index a5f2331a068..7b0cfdf6a6c 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp index 45d667047af..8ea732b0243 100644 --- a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp @@ -1,11 +1,12 @@ -#include -#include #include #include +#include +#include #include #include #include -#include +#include +#include namespace DB diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index eec2f53381f..4406a7c3fd4 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 6e7ac2b47b8..e4cd166437e 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -24,8 +25,8 @@ #include #include #include -#include +#include namespace fs = std::filesystem; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 9958d65819b..16428d2dc1c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -5,14 +5,15 @@ #include #include +#include #include +#include #include #include -#include #include -#include +#include #include -#include +#include #include #include diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 5e4fde99306..7a8852ca2d6 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -2,8 +2,9 @@ #include #include -#include #include +#include +#include #include diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 3c01b4717cc..cb8d5caa50c 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 5c96c6502af..8c6d29a3b70 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp index f5537b508ba..2351c3c6a2a 100644 --- a/src/Storages/System/StorageSystemDatabases.cpp +++ b/src/Storages/System/StorageSystemDatabases.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 50c6436f316..e2058448904 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace DB diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 20baeee1d3b..344c653f7e4 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index eaa386763c2..d8b760e1302 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 60b80e0b0ad..94656008029 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..812a4e6efbe 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 639c1455b83..edfc7213dcd 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp index 8041370ee92..950e20512c0 100644 --- a/src/Storages/System/StorageSystemZooKeeperConnection.cpp +++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp @@ -8,6 +8,8 @@ #include #include +#include + namespace DB { diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index c87a1b216ca..41c8f475f03 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index c545367b63d..26e953c0578 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index ff1459ca34b..13b5c167e0e 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include From ec6c276a92a3f9d340e3fd1ac80a2db06c3e7ccb Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 18 Mar 2024 18:02:54 +0300 Subject: [PATCH 173/332] Temporary data use temporary_files_codec setting --- src/Compression/CompressionFactory.cpp | 8 +++++ src/Compression/CompressionFactory.h | 3 ++ src/Core/Settings.h | 2 +- src/Interpreters/Context.cpp | 15 ++++++--- src/Interpreters/ProcessList.cpp | 18 ++++++++-- src/Interpreters/TemporaryDataOnDisk.cpp | 24 +++++++++----- src/Interpreters/TemporaryDataOnDisk.h | 33 ++++++++++++++----- src/Interpreters/tests/gtest_filecache.cpp | 6 ++-- .../MergeTreeDataPartWriterCompact.cpp | 9 +---- .../MergeTreeDataPartWriterOnDisk.cpp | 4 +-- 10 files changed, 84 insertions(+), 38 deletions(-) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index f4413401667..68e0131c91b 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -44,6 +46,12 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std } } +CompressionCodecPtr CompressionCodecFactory::get(const String & compression_codec) const +{ + ParserCodec codec_parser; + auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + return CompressionCodecFactory::instance().get(ast, nullptr); +} CompressionCodecPtr CompressionCodecFactory::get( const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index e71476d564d..2885f35d7bd 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -68,6 +68,9 @@ public: /// For backward compatibility with config settings CompressionCodecPtr get(const String & family_name, std::optional level) const; + /// Get codec by name with optional params. Example: LZ4, ZSTD(3) + CompressionCodecPtr get(const String & compression_codec) const; + /// Register codec with parameters and column type void registerCompressionCodecWithType(const String & family_name, std::optional byte_code, CreatorWithType creator); /// Register codec with parameters diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a3c5638d97f..a04a8e430e9 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -450,7 +450,7 @@ class IColumn; \ M(Bool, compatibility_ignore_collation_in_create_table, true, "Compatibility ignore collation in create table", 0) \ \ - M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files (sort and join on disk). I.e. LZ4, NONE.", 0) \ + M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files produced by (JOINs, external GROUP BY, external ORDER BY). I.e. LZ4, NONE.", 0) \ \ M(UInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ M(UInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6a0657a842c..038b9712b0c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1073,7 +1073,9 @@ void Context::setTemporaryStoragePath(const String & path, size_t max_size) setupTmpPath(shared->log, disk->getPath()); } - shared->root_temp_data_on_disk = std::make_shared(volume, max_size); + TemporaryDataOnDiskSettings temporary_data_on_disk_settings; + temporary_data_on_disk_settings.max_size_on_disk = max_size; + shared->root_temp_data_on_disk = std::make_shared(std::move(volume), std::move(temporary_data_on_disk_settings)); } void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_size) @@ -1093,7 +1095,7 @@ void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_s VolumePtr volume = tmp_policy->getVolume(0); if (volume->getDisks().empty()) - throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "No disks volume for temporary files"); + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "No disks volume for temporary files"); for (const auto & disk : volume->getDisks()) { @@ -1119,7 +1121,9 @@ void Context::setTemporaryStoragePolicy(const String & policy_name, size_t max_s if (shared->root_temp_data_on_disk) throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary storage is already set"); - shared->root_temp_data_on_disk = std::make_shared(volume, max_size); + TemporaryDataOnDiskSettings temporary_data_on_disk_settings; + temporary_data_on_disk_settings.max_size_on_disk = max_size; + shared->root_temp_data_on_disk = std::make_shared(std::move(volume), std::move(temporary_data_on_disk_settings)); } void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t max_size) @@ -1140,7 +1144,10 @@ void Context::setTemporaryStorageInCache(const String & cache_disk_name, size_t shared->tmp_path = file_cache->getBasePath(); VolumePtr volume = createLocalSingleDiskVolume(shared->tmp_path, shared->getConfigRefWithLock(lock)); - shared->root_temp_data_on_disk = std::make_shared(volume, file_cache.get(), max_size); + + TemporaryDataOnDiskSettings temporary_data_on_disk_settings; + temporary_data_on_disk_settings.max_size_on_disk = max_size; + shared->root_temp_data_on_disk = std::make_shared(std::move(volume), file_cache.get(), std::move(temporary_data_on_disk_settings)); } void Context::setFlagsPath(const String & path) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3bd7b2d4206..889ad79f4ef 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -208,8 +208,13 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker); if (user_process_list.user_temp_data_on_disk) { + TemporaryDataOnDiskSettings temporary_data_on_disk_settings + { + .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_query, + .compression_codec = settings.temporary_files_codec + }; query_context->setTempDataOnDisk(std::make_shared( - user_process_list.user_temp_data_on_disk, settings.max_temporary_data_on_disk_size_for_query)); + user_process_list.user_temp_data_on_disk, std::move(temporary_data_on_disk_settings))); } /// Set query-level memory trackers @@ -682,8 +687,15 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList * if (global_context) { - size_t size_limit = global_context->getSettingsRef().max_temporary_data_on_disk_size_for_user; - user_temp_data_on_disk = std::make_shared(global_context->getSharedTempDataOnDisk(), size_limit); + auto & settings = global_context->getSettingsRef(); + TemporaryDataOnDiskSettings temporary_data_on_disk_settings + { + .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_user, + .compression_codec = settings.temporary_files_codec + }; + + user_temp_data_on_disk = std::make_shared(global_context->getSharedTempDataOnDisk(), + std::move(temporary_data_on_disk_settings)); } } diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index a48e7d8e040..4a30c0ae726 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -44,20 +44,20 @@ void TemporaryDataOnDiskScope::deltaAllocAndCheck(ssize_t compressed_delta, ssiz } size_t new_consumprion = stat.compressed_size + compressed_delta; - if (compressed_delta > 0 && limit && new_consumprion > limit) + if (compressed_delta > 0 && settings.max_size_on_disk && new_consumprion > settings.max_size_on_disk) throw Exception(ErrorCodes::TOO_MANY_ROWS_OR_BYTES, - "Limit for temporary files size exceeded (would consume {} / {} bytes)", new_consumprion, limit); + "Limit for temporary files size exceeded (would consume {} / {} bytes)", new_consumprion, settings.max_size_on_disk); stat.compressed_size += compressed_delta; stat.uncompressed_size += uncompressed_delta; } TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_) - : TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0) + : TemporaryDataOnDiskScope(parent_, parent_->getSettings()) {} TemporaryDataOnDisk::TemporaryDataOnDisk(TemporaryDataOnDiskScopePtr parent_, CurrentMetrics::Metric metric_scope) - : TemporaryDataOnDiskScope(std::move(parent_), /* limit_ = */ 0) + : TemporaryDataOnDiskScope(parent_, parent_->getSettings()) , current_metric_scope(metric_scope) {} @@ -153,11 +153,19 @@ bool TemporaryDataOnDisk::empty() const return streams.empty(); } +static inline CompressionCodecPtr getCodec(const TemporaryDataOnDiskSettings & settings) +{ + if (settings.compression_codec.empty()) + return CompressionCodecFactory::instance().get("NONE"); + + return CompressionCodecFactory::instance().get(settings.compression_codec); +} + struct TemporaryFileStream::OutputWriter { - OutputWriter(std::unique_ptr out_buf_, const Block & header_) + OutputWriter(std::unique_ptr out_buf_, const Block & header_, const TemporaryDataOnDiskSettings & settings) : out_buf(std::move(out_buf_)) - , out_compressed_buf(*out_buf) + , out_compressed_buf(*out_buf, getCodec(settings)) , out_writer(out_compressed_buf, DBMS_TCP_PROTOCOL_VERSION, header_) { } @@ -248,7 +256,7 @@ TemporaryFileStream::TemporaryFileStream(TemporaryFileOnDiskHolder file_, const : parent(parent_) , header(header_) , file(std::move(file_)) - , out_writer(std::make_unique(std::make_unique(file->getAbsolutePath()), header)) + , out_writer(std::make_unique(std::make_unique(file->getAbsolutePath()), header, parent->settings)) { LOG_TEST(getLogger("TemporaryFileStream"), "Writing to temporary file {}", file->getAbsolutePath()); } @@ -263,7 +271,7 @@ TemporaryFileStream::TemporaryFileStream(FileSegmentsHolderPtr segments_, const auto out_buf = std::make_unique(&segment_holder->front()); LOG_TEST(getLogger("TemporaryFileStream"), "Writing to temporary file {}", out_buf->getFileName()); - out_writer = std::make_unique(std::move(out_buf), header); + out_writer = std::make_unique(std::move(out_buf), header, parent_->settings); } size_t TemporaryFileStream::write(const Block & block) diff --git a/src/Interpreters/TemporaryDataOnDisk.h b/src/Interpreters/TemporaryDataOnDisk.h index 8b0649be1b1..40100a62b44 100644 --- a/src/Interpreters/TemporaryDataOnDisk.h +++ b/src/Interpreters/TemporaryDataOnDisk.h @@ -28,6 +28,15 @@ using TemporaryFileStreamPtr = std::unique_ptr; class FileCache; +struct TemporaryDataOnDiskSettings +{ + /// Max size on disk, if 0 there will be no limit + size_t max_size_on_disk = 0; + + /// Compression codec for temporary data, if empty no compression will be used. LZ4 by default + String compression_codec = "LZ4"; +}; + /* * Used to account amount of temporary data written to disk. * If limit is set, throws exception if limit is exceeded. @@ -43,22 +52,30 @@ public: std::atomic uncompressed_size; }; - explicit TemporaryDataOnDiskScope(VolumePtr volume_, size_t limit_) - : volume(std::move(volume_)), limit(limit_) + explicit TemporaryDataOnDiskScope(VolumePtr volume_, TemporaryDataOnDiskSettings settings_) + : volume(std::move(volume_)) + , settings(std::move(settings_)) {} - explicit TemporaryDataOnDiskScope(VolumePtr volume_, FileCache * file_cache_, size_t limit_) - : volume(std::move(volume_)), file_cache(file_cache_), limit(limit_) + explicit TemporaryDataOnDiskScope(VolumePtr volume_, FileCache * file_cache_, TemporaryDataOnDiskSettings settings_) + : volume(std::move(volume_)) + , file_cache(file_cache_) + , settings(std::move(settings_)) {} - explicit TemporaryDataOnDiskScope(TemporaryDataOnDiskScopePtr parent_, size_t limit_) - : parent(std::move(parent_)), volume(parent->volume), file_cache(parent->file_cache), limit(limit_) + explicit TemporaryDataOnDiskScope(TemporaryDataOnDiskScopePtr parent_, TemporaryDataOnDiskSettings settings_) + : parent(std::move(parent_)) + , volume(parent->volume) + , file_cache(parent->file_cache) + , settings(std::move(settings_)) {} /// TODO: remove /// Refactor all code that uses volume directly to use TemporaryDataOnDisk. VolumePtr getVolume() const { return volume; } + const TemporaryDataOnDiskSettings & getSettings() const { return settings; } + protected: void deltaAllocAndCheck(ssize_t compressed_delta, ssize_t uncompressed_delta); @@ -68,14 +85,14 @@ protected: FileCache * file_cache = nullptr; StatAtomic stat; - size_t limit = 0; + const TemporaryDataOnDiskSettings settings; }; /* * Holds the set of temporary files. * New file stream is created with `createStream`. * Streams are owned by this object and will be deleted when it is deleted. - * It's a leaf node in temorarty data scope tree. + * It's a leaf node in temporary data scope tree. */ class TemporaryDataOnDisk : private TemporaryDataOnDiskScope { diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index 2679d1b8d18..09afb01d7ff 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -947,7 +947,7 @@ TEST_F(FileCacheTest, temporaryData) file_cache.initialize(); const auto user = FileCache::getCommonUser(); - auto tmp_data_scope = std::make_shared(nullptr, &file_cache, 0); + auto tmp_data_scope = std::make_shared(nullptr, &file_cache, TemporaryDataOnDiskSettings{}); auto some_data_holder = file_cache.getOrSet(file_cache.createKeyForPath("some_data"), 0, 5_KiB, 5_KiB, CreateFileSegmentSettings{}, 0, user); @@ -1130,7 +1130,7 @@ TEST_F(FileCacheTest, TemporaryDataReadBufferSize) DB::FileCache file_cache("cache", settings); file_cache.initialize(); - auto tmp_data_scope = std::make_shared(/*volume=*/nullptr, &file_cache, /*limit=*/0); + auto tmp_data_scope = std::make_shared(/*volume=*/nullptr, &file_cache, /*settings=*/TemporaryDataOnDiskSettings{}); auto tmp_data = std::make_unique(tmp_data_scope); @@ -1152,7 +1152,7 @@ TEST_F(FileCacheTest, TemporaryDataReadBufferSize) disk = createDisk("temporary_data_read_buffer_size_test_dir"); VolumePtr volume = std::make_shared("volume", disk); - auto tmp_data_scope = std::make_shared(/*volume=*/volume, /*cache=*/nullptr, /*limit=*/0); + auto tmp_data_scope = std::make_shared(/*volume=*/volume, /*cache=*/nullptr, /*settings=*/TemporaryDataOnDiskSettings{}); auto tmp_data = std::make_unique(tmp_data_scope); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index fe45d0bee54..1605e5cdb9a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -9,13 +9,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -static CompressionCodecPtr getMarksCompressionCodec(const String & marks_compression_codec) -{ - ParserCodec codec_parser; - auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(marks_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - return CompressionCodecFactory::instance().get(ast, nullptr); -} - MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const MergeTreeMutableDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, @@ -46,7 +39,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( { marks_compressor = std::make_unique( *marks_file_hashing, - getMarksCompressionCodec(settings_.marks_compression_codec), + CompressionCodecFactory::instance().get(settings_.marks_compression_codec), settings_.marks_compress_block_size); marks_source_hashing = std::make_unique(*marks_compressor); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index a31da5bc4fe..5a82357c6eb 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -242,9 +242,7 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex() if (compress_primary_key) { - ParserCodec codec_parser; - auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(settings.primary_key_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); - CompressionCodecPtr primary_key_compression_codec = CompressionCodecFactory::instance().get(ast, nullptr); + CompressionCodecPtr primary_key_compression_codec = CompressionCodecFactory::instance().get(settings.primary_key_compression_codec); index_compressor_stream = std::make_unique(*index_file_hashing_stream, primary_key_compression_codec, settings.primary_key_compress_block_size); index_source_hashing_stream = std::make_unique(*index_compressor_stream); } From f4d20a79b08525a3d39716b9a95b6f3ae6058817 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 19 Mar 2024 19:07:02 +0300 Subject: [PATCH 174/332] Fixed build --- src/Interpreters/ProcessList.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 889ad79f4ef..0e2a6e14f07 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -687,7 +687,7 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList * if (global_context) { - auto & settings = global_context->getSettingsRef(); + const auto & settings = global_context->getSettingsRef(); TemporaryDataOnDiskSettings temporary_data_on_disk_settings { .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_user, From 574647ed2a9e78185a9dcf9d104d6f73d2588700 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:16:53 +0100 Subject: [PATCH 175/332] Update config for another test --- tests/integration/test_mask_sensitive_info/configs/users.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml index f129a5bb3e3..54ee9b2c9db 100644 --- a/tests/integration/test_mask_sensitive_info/configs/users.xml +++ b/tests/integration/test_mask_sensitive_info/configs/users.xml @@ -2,6 +2,7 @@ 5 + 1 From 79bbcbdf1e438d98298499d0c007c7b52b979667 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:20:31 +0100 Subject: [PATCH 176/332] Update StorageS3Queue.cpp --- src/Storages/S3Queue/StorageS3Queue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 048816fe4f1..bf4d3759d8e 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -133,7 +133,7 @@ StorageS3Queue::StorageS3Queue( if (mode == LoadingStrictnessLevel::CREATE && !context_->getSettingsRef().s3queue_allow_experimental_sharded_mode && s3queue_settings->mode == S3QueueMode::ORDERED - && s3queue_settings->s3queue_total_shards_num) + && (s3queue_settings->s3queue_total_shards_num > 1 || s3queue_settings->s3queue_processing_threads_num > 1)) { throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue sharded mode is not allowed. To enable use `s3queue_allow_experimental_sharded_mode`"); } From 7ebab45beff7bdea5c2dbef2e1aaa568c7c7fbca Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:21:03 +0100 Subject: [PATCH 177/332] Update users.xml --- tests/integration/test_mask_sensitive_info/configs/users.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml index 54ee9b2c9db..f129a5bb3e3 100644 --- a/tests/integration/test_mask_sensitive_info/configs/users.xml +++ b/tests/integration/test_mask_sensitive_info/configs/users.xml @@ -2,7 +2,6 @@ 5 - 1 From c6f0a434c04023290b67403eeab7136631573798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 17:40:27 +0100 Subject: [PATCH 178/332] Remove some magic_enum from headers --- src/Common/IntervalKind.cpp | 7 +++ src/Common/IntervalKind.h | 3 +- .../MergeTree/MergeTreeDataPartType.cpp | 47 +++++++++++++++++++ .../MergeTree/MergeTreeDataPartType.h | 28 ++--------- 4 files changed, 59 insertions(+), 26 deletions(-) create mode 100644 src/Storages/MergeTree/MergeTreeDataPartType.cpp diff --git a/src/Common/IntervalKind.cpp b/src/Common/IntervalKind.cpp index 7ba32a689f5..22c7db504c3 100644 --- a/src/Common/IntervalKind.cpp +++ b/src/Common/IntervalKind.cpp @@ -1,6 +1,8 @@ #include #include +#include + namespace DB { @@ -10,6 +12,11 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +std::string_view IntervalKind::toString() const +{ + return magic_enum::enum_name(kind); +} + Int64 IntervalKind::toAvgNanoseconds() const { static constexpr Int64 NANOSECONDS_PER_MICROSECOND = 1000; diff --git a/src/Common/IntervalKind.h b/src/Common/IntervalKind.h index d77a39cb9fa..497dc83be0b 100644 --- a/src/Common/IntervalKind.h +++ b/src/Common/IntervalKind.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB { @@ -27,7 +26,7 @@ struct IntervalKind IntervalKind(Kind kind_ = Kind::Second) : kind(kind_) {} /// NOLINT operator Kind() const { return kind; } /// NOLINT - constexpr std::string_view toString() const { return magic_enum::enum_name(kind); } + std::string_view toString() const; /// Returns number of nanoseconds in one interval. /// For `Month`, `Quarter` and `Year` the function returns an average number of nanoseconds. diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.cpp b/src/Storages/MergeTree/MergeTreeDataPartType.cpp new file mode 100644 index 00000000000..fb5555cb13f --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeDataPartType.cpp @@ -0,0 +1,47 @@ +#include +#include + +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} + +template +requires std::is_enum_v +static E parseEnum(const String & str) +{ + auto value = magic_enum::enum_cast(str); + if (!value || *value == E::Unknown) + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected string {} for enum {}", str, magic_enum::enum_type_name()); + + return *value; +} + +String MergeTreeDataPartType::toString() const +{ + return String(magic_enum::enum_name(value)); +} + +void MergeTreeDataPartType::fromString(const String & str) +{ + value = parseEnum(str); +} + +String MergeTreeDataPartStorageType::toString() const +{ + return String(magic_enum::enum_name(value)); +} + +void MergeTreeDataPartStorageType::fromString(const String & str) +{ + value = parseEnum(str); +} + +} diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.h b/src/Storages/MergeTree/MergeTreeDataPartType.h index 64f3abee391..8177809d41e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -1,29 +1,9 @@ #pragma once -#include #include -#include namespace DB { - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - -template -requires std::is_enum_v -static E parseEnum(const String & str) -{ - auto value = magic_enum::enum_cast(str); - if (!value || *value == E::Unknown) - throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected string {} for enum {}", str, magic_enum::enum_type_name()); - - return *value; -} - /// It's a bug in clang with three-way comparison operator /// https://github.com/llvm/llvm-project/issues/55919 #pragma clang diagnostic push @@ -51,8 +31,8 @@ public: auto operator<=>(const MergeTreeDataPartType &) const = default; Value getValue() const { return value; } - String toString() const { return String(magic_enum::enum_name(value)); } - void fromString(const String & str) { value = parseEnum(str); } + String toString() const; + void fromString(const String & str); private: Value value; @@ -74,8 +54,8 @@ public: auto operator<=>(const MergeTreeDataPartStorageType &) const = default; Value getValue() const { return value; } - String toString() const { return String(magic_enum::enum_name(value)); } - void fromString(const String & str) { value = parseEnum(str); } + String toString() const; + void fromString(const String & str); private: Value value; From 764c72b6c9bc3c54d7e505514aef8b8c4c7a5fd7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 19 Mar 2024 20:41:39 +0300 Subject: [PATCH 179/332] Fixed code review issues --- tests/integration/test_temporary_data_in_cache/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py index ed06a70cf5a..5eed7563d9b 100644 --- a/tests/integration/test_temporary_data_in_cache/test.py +++ b/tests/integration/test_temporary_data_in_cache/test.py @@ -67,6 +67,7 @@ def test_cache_evicted_by_temporary_data(start_cluster): settings={ "max_bytes_before_external_group_by": "4M", "max_bytes_before_external_sort": "4M", + "temporary_files_codec": "ZSTD" }, ) assert fnmatch.fnmatch( From f60b5f3cb7f1c32fd6f5d9837c4d27401dbe5da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 18:45:07 +0100 Subject: [PATCH 180/332] Add test for #60317 --- .../03014_analyzer_groupby_fuzz_60317.reference | 1 + .../0_stateless/03014_analyzer_groupby_fuzz_60317.sql | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference create mode 100644 tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference new file mode 100644 index 00000000000..4972904f87d --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference @@ -0,0 +1 @@ +30 30 1970-01-01 diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql new file mode 100644 index 00000000000..2f54058526e --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql @@ -0,0 +1,9 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/60317 +SELECT + toNullable(materialize(_CAST(30, 'LowCardinality(UInt8)'))) as a, + _CAST(30, 'LowCardinality(UInt8)') as b, + makeDate(materialize(_CAST(30, 'LowCardinality(UInt8)')), 10, _CAST(30, 'Nullable(UInt8)')) as c +FROM system.one +GROUP BY + _CAST(30, 'Nullable(UInt8)') +SETTINGS allow_experimental_analyzer = 1; From e954f684823c98adfe5bc99af8de4c7a615a60a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 18:51:09 +0100 Subject: [PATCH 181/332] Add test for #60772 --- .../03015_analyzer_groupby_fuzz_60772.reference | 1 + .../0_stateless/03015_analyzer_groupby_fuzz_60772.sql | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference create mode 100644 tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference new file mode 100644 index 00000000000..611407ecd90 --- /dev/null +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference @@ -0,0 +1 @@ +%W 2018-01-02 22:33:44 1 diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql new file mode 100644 index 00000000000..c8b4eef50ff --- /dev/null +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/60772 +SELECT toFixedString(toFixedString(toFixedString(toFixedString(toFixedString(toFixedString('%W', 2), 2), 2),toLowCardinality(toLowCardinality(toNullable(2)))), 2), 2), + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + hasSubsequence(toNullable(materialize(toLowCardinality('garbage'))), 'gr') +GROUP BY + '2018-01-02 22:33:44', + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + 'gr', + '2018-01-02 22:33:44' +SETTINGS allow_experimental_analyzer = 1; From b19540f5b6f6b229a4c49353a01d8cc8921b5602 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 19 Mar 2024 19:03:20 +0100 Subject: [PATCH 182/332] Fix style tests/integration/test_temporary_data_in_cache/test.py --- tests/integration/test_temporary_data_in_cache/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py index 5eed7563d9b..cab134dcce2 100644 --- a/tests/integration/test_temporary_data_in_cache/test.py +++ b/tests/integration/test_temporary_data_in_cache/test.py @@ -67,7 +67,7 @@ def test_cache_evicted_by_temporary_data(start_cluster): settings={ "max_bytes_before_external_group_by": "4M", "max_bytes_before_external_sort": "4M", - "temporary_files_codec": "ZSTD" + "temporary_files_codec": "ZSTD", }, ) assert fnmatch.fnmatch( From d56838080011e65ac2363862e8d6157bf2b35d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 19:03:35 +0100 Subject: [PATCH 183/332] Add test for #59796 --- .../0_stateless/03016_analyzer_groupby_fuzz_59796.reference | 1 + .../0_stateless/03016_analyzer_groupby_fuzz_59796.sql | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference create mode 100644 tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql diff --git a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference new file mode 100644 index 00000000000..6f7389b482b --- /dev/null +++ b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.reference @@ -0,0 +1 @@ +\0\0\0 key="v" \0 key="v" key="v" key="v" key="v" \0 key="v" diff --git a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql new file mode 100644 index 00000000000..c00a75a631c --- /dev/null +++ b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql @@ -0,0 +1,6 @@ +SELECT + concat(concat(unhex('00'), concat(unhex('00'), concat(unhex(toFixedString('00', 2)), toFixedString(toFixedString(' key="v" ', 9), 9), concat(unhex('00'), toFixedString(' key="v" ', 9)), toFixedString(materialize(toLowCardinality(' key="v" ')), 9)), toFixedString(' key="v" ', 9)), toFixedString(' key="v" ', 9)), unhex('00'), ' key="v" ') AS haystack +GROUP BY + concat(unhex('00'), toFixedString(materialize(toFixedString(' key="v" ', 9)), 9), toFixedString(toFixedString('00', 2), toNullable(2)), toFixedString(toFixedString(toFixedString(' key="v" ', 9), 9), 9)), + concat(' key="v" ') +SETTINGS allow_experimental_analyzer = 1; From 73d601d231a8825643492b8895e6d1b43f04915f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 19 Mar 2024 19:11:32 +0100 Subject: [PATCH 184/332] Add test for #61600 --- .../03017_analyzer_groupby_fuzz_61600.reference | 1 + .../03017_analyzer_groupby_fuzz_61600.sql | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference create mode 100644 tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference new file mode 100644 index 00000000000..68acb650f8e --- /dev/null +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference @@ -0,0 +1 @@ +38 \N diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql new file mode 100644 index 00000000000..44b27750c16 --- /dev/null +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql @@ -0,0 +1,14 @@ +CREATE TABLE set_index_not__fuzz_0 (`name` String, `status` Enum8('alive' = 0, 'rip' = 1), INDEX idx_status status TYPE set(2) GRANULARITY 1) +ENGINE = MergeTree ORDER BY name +SETTINGS index_granularity = 8192; + +INSERT INTO set_index_not__fuzz_0 SELECT * from generateRandom() limit 1; + +SELECT + 38, + concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3),NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3))) +FROM set_index_not__fuzz_0 +GROUP BY + toNullable(3), + concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) +SETTINGS allow_experimental_analyzer = 1; From 221b74c77fd56fd184b770620af2c634f5227680 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 19 Mar 2024 19:44:15 +0100 Subject: [PATCH 185/332] Fix #ci_set_analyzer --- src/Analyzer/QueryNode.h | 5 +++++ src/Interpreters/ClusterProxy/executeQuery.cpp | 1 + src/Planner/PlannerJoinTree.cpp | 7 +++++++ src/Storages/StorageDistributed.cpp | 2 ++ src/Storages/buildQueryTreeForShard.cpp | 7 +++++++ 5 files changed, 22 insertions(+) diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index d8b8741afb2..6f9067908dd 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -99,6 +99,11 @@ public: return settings_changes; } + void clearSettingsChanges() + { + settings_changes.clear(); + } + /// Returns true if query node is subquery, false otherwise bool isSubquery() const { diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 07ef7aa6c96..2348081b56c 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -127,6 +127,7 @@ ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, /// /// Here we don't try to analyze setting again. In case if query_info->additional_filter_ast is not empty, some filter was applied. /// It's just easier to add this filter for a source table. + LOG_DEBUG(&Poco::Logger::get("updateSettings"), "{}", additional_filter_ast != nullptr); if (additional_filter_ast) { Tuple tuple; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 8ca8f0f258b..e7afd7a1ea4 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,5 +1,6 @@ #include +#include "Common/logger_useful.h" #include #include @@ -63,6 +64,7 @@ #include #include #include +#include namespace DB { @@ -523,6 +525,7 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, return {}; auto const & storage_id = storage->getStorageID(); + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", storage_id.getFullNameNotQuoted()); ASTPtr additional_filter_ast; for (const auto & additional_filter : additional_filters) @@ -543,6 +546,8 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, } } + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filter_ast != nullptr); + if (!additional_filter_ast) return {}; @@ -847,6 +852,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const auto & table_expression_alias = table_expression->getOriginalAlias(); auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context); + if (additional_filters_info.actions) + LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filters_info.actions->dumpDAG()); add_filter(additional_filters_info, "additional filter"); from_stage = storage->getQueryProcessingStage( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index a9b909bc71e..675e5b1071a 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -836,6 +836,8 @@ void StorageDistributed::read( { Block header; + LOG_DEBUG(&Poco::Logger::get("read"), "{}", query_info.additional_filter_ast != nullptr); + SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index c87a1b216ca..ff88172e7a3 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "Analyzer/QueryNode.h" namespace DB { @@ -383,6 +384,12 @@ QueryTreeNodePtr buildQueryTreeForShard(const PlannerContextPtr & planner_contex createUniqueTableAliases(query_tree_to_modify, nullptr, planner_context->getQueryContext()); + // Get rid of the settings clause so we don't send them to remote. Thus newly non-important + // settings won't break any remote parser. It's also more reasonable since the query settings + // are written into the query context and will be sent by the query pipeline. + if (auto * query_node = query_tree_to_modify->as()) + query_node->clearSettingsChanges(); + return query_tree_to_modify; } From b561560c9968cacef84b62564e2dbf37182bcb6f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 19 Mar 2024 19:55:59 +0100 Subject: [PATCH 186/332] Remove it from remaining places --- src/Common/SensitiveDataMasker.cpp | 2 +- .../JoinToSubqueryTransformVisitor.cpp | 8 ++-- .../TranslateQualifiedNamesVisitor.cpp | 8 ++-- src/Parsers/ASTColumnsMatcher.cpp | 6 --- src/Parsers/ASTColumnsTransformers.cpp | 40 +++++++++---------- src/Parsers/ASTColumnsTransformers.h | 9 ++--- .../Formats/Impl/AvroRowOutputFormat.cpp | 2 +- 7 files changed, 33 insertions(+), 42 deletions(-) diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 28eae6f451d..8c29b899841 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -60,7 +60,7 @@ public: , replacement(replacement_string) { if (!regexp.ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "SensitiveDataMasker: cannot compile re2: {}, error: {}. " "Look at https://github.com/google/re2/wiki/Syntax for reference.", regexp_string_, regexp.error()); diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 209f8e68b8f..5cda4c982b4 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -205,16 +205,16 @@ private: has_asterisks = true; String pattern = columns_regexp_matcher->getPattern(); - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); - if (!regexp->ok()) + re2::RE2 regexp(pattern, re2::RE2::Quiet); + if (!regexp.ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp.error()); for (auto & table_name : data.tables_order) data.addTableColumns( table_name, columns, - [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, *regexp); }); + [&](const String & column_name) { return re2::RE2::PartialMatch(column_name, regexp); }); if (columns_regexp_matcher->transformers) { diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index e3690d0fcc0..03df7283992 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -278,17 +278,17 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt else if (const auto * asterisk_regexp_pattern = child->as()) { String pattern = asterisk_regexp_pattern->getPattern(); - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); - if (!regexp->ok()) + re2::RE2 regexp(pattern, re2::RE2::Quiet); + if (!regexp.ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp.error()); bool first_table = true; for (const auto & table : tables_with_columns) { for (const auto & column : table.columns) { - if (re2::RE2::PartialMatch(column.name, *regexp) + if (re2::RE2::PartialMatch(column.name, regexp) && (first_table || !data.join_using_columns.contains(column.name))) { addIdentifier(columns, table.table, column.name); diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index dac3661d482..518fd9b722d 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -4,17 +4,11 @@ #include #include #include -#include namespace DB { -namespace ErrorCodes -{ - extern const int CANNOT_COMPILE_REGEXP; -} - ASTPtr ASTColumnsRegexpMatcher::clone() const { auto clone = std::make_shared(*this); diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 34a1ae6e8e0..7f3fda9914c 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -12,6 +12,7 @@ namespace DB { + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -179,8 +180,8 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo (*it)->formatImpl(settings, state, frame); } - if (!original_pattern.empty()) - settings.ostr << quoteString(original_pattern); + if (!pattern.empty()) + settings.ostr << quoteString(pattern); if (children.size() > 1) settings.ostr << ")"; @@ -202,8 +203,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const (*it)->appendColumnName(ostr); } - if (!original_pattern.empty()) - writeQuotedString(original_pattern, ostr); + if (!pattern.empty()) + writeQuotedString(pattern, ostr); if (children.size() > 1) writeChar(')', ostr); @@ -212,8 +213,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { hash_state.update(is_strict); - hash_state.update(original_pattern.size()); - hash_state.update(original_pattern); + hash_state.update(pattern.size()); + hash_state.update(pattern); IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -221,7 +222,7 @@ void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool void ASTColumnsExceptTransformer::transform(ASTs & nodes) const { std::set expected_columns; - if (original_pattern.empty()) + if (pattern.empty()) { for (const auto & child : children) expected_columns.insert(child->as().name()); @@ -243,11 +244,13 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const } else { + auto regexp = getMatcher(); + for (auto * it = nodes.begin(); it != nodes.end();) { if (const auto * id = it->get()->as()) { - if (isColumnMatching(id->shortName())) + if (RE2::PartialMatch(id->shortName(), *regexp)) { it = nodes.erase(it); continue; @@ -268,23 +271,18 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const } } -void ASTColumnsExceptTransformer::setPattern(String pattern) +void ASTColumnsExceptTransformer::setPattern(String pattern_) { - original_pattern = std::move(pattern); - column_matcher = std::make_shared(original_pattern, RE2::Quiet); - if (!column_matcher->ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "COLUMNS pattern {} cannot be compiled: {}", - original_pattern, column_matcher->error()); + pattern = std::move(pattern_); } -const std::shared_ptr & ASTColumnsExceptTransformer::getMatcher() const +std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { - return column_matcher; -} - -bool ASTColumnsExceptTransformer::isColumnMatching(const String & column_name) const -{ - return RE2::PartialMatch(column_name, *column_matcher); + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + if (!regexp->ok()) + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + return regexp; } void ASTColumnsReplaceTransformer::Replacement::formatImpl( diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index a2a138e13c9..2d13cd85819 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -2,6 +2,7 @@ #include + namespace re2 { class RE2; @@ -76,16 +77,14 @@ public: return clone; } void transform(ASTs & nodes) const override; - void setPattern(String pattern); - const std::shared_ptr & getMatcher() const; - bool isColumnMatching(const String & column_name) const; + void setPattern(String pattern_); + std::shared_ptr getMatcher() const; void appendColumnName(WriteBuffer & ostr) const override; void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override; protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - std::shared_ptr column_matcher; - String original_pattern; + String pattern; }; class ASTColumnsReplaceTransformer : public IASTColumnsTransformer diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 07700942b57..a79a7d10c78 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -52,7 +52,7 @@ public: : string_to_string_regexp(settings_.avro.string_column_pattern) { if (!string_to_string_regexp.ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " + throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " "Look at https://github.com/google/re2/wiki/Syntax for reference.", settings_.avro.string_column_pattern, string_to_string_regexp.error()); } From 1913466da4487784582df7301fe10464a80b2a4a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 Mar 2024 20:31:00 +0100 Subject: [PATCH 187/332] fix uaf of async_insert_queue --- src/Interpreters/AsynchronousInsertQueue.cpp | 15 ++++++++++++++- src/Interpreters/AsynchronousInsertQueue.h | 2 ++ src/Interpreters/Context.cpp | 8 +++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 9234d052d97..e82a902e03b 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -218,7 +218,7 @@ AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t poo dump_by_first_update_threads.emplace_back([this, i] { processBatchDeadlines(i); }); } -AsynchronousInsertQueue::~AsynchronousInsertQueue() +void AsynchronousInsertQueue::flushAndShutdown() { try { @@ -257,6 +257,19 @@ AsynchronousInsertQueue::~AsynchronousInsertQueue() } } +AsynchronousInsertQueue::~AsynchronousInsertQueue() +{ + for (const auto & shard : queue_shards) + { + for (const auto & [first_update, elem] : shard.queue) + { + const auto & insert_query = elem.key.query->as(); + LOG_WARNING(log, "Has unprocessed async insert for {}.{}", + backQuoteIfNeed(insert_query.getDatabase()), backQuoteIfNeed(insert_query.getTable())); + } + } +} + void AsynchronousInsertQueue::scheduleDataProcessingJob( const InsertQuery & key, InsertDataPtr data, ContextPtr global_context, size_t shard_num) { diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 1a006cce85c..1a4678e3a7d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -63,6 +63,8 @@ public: PushResult pushQueryWithBlock(ASTPtr query, Block block, ContextPtr query_context); size_t getPoolSize() const { return pool_size; } + void flushAndShutdown(); + private: struct InsertQuery diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b1d2621e925..f388cf6b3de 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -566,7 +566,7 @@ struct ContextSharedPart : boost::noncopyable std::lock_guard lock(mutex); delete_async_insert_queue = std::move(async_insert_queue); } - delete_async_insert_queue.reset(); + delete_async_insert_queue->flushAndShutdown(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config @@ -590,6 +590,8 @@ struct ContextSharedPart : boost::noncopyable LOG_TRACE(log, "Shutting down database catalog"); DatabaseCatalog::shutdown(); + delete_async_insert_queue.reset(); + SHUTDOWN(log, "merges executor", merge_mutate_executor, wait()); SHUTDOWN(log, "fetches executor", fetch_executor, wait()); SHUTDOWN(log, "moves executor", moves_executor, wait()); @@ -4990,7 +4992,7 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const AsynchronousInsertQueue * Context::tryGetAsynchronousInsertQueue() const { - std::lock_guard lock(mutex); + SharedLockGuard lock(shared->mutex); return shared->async_insert_queue.get(); } @@ -4998,7 +5000,7 @@ void Context::setAsynchronousInsertQueue(const std::shared_ptrmutex); if (std::chrono::milliseconds(settings.async_insert_poll_timeout_ms) == std::chrono::milliseconds::zero()) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting async_insert_poll_timeout_ms can't be zero"); From 2f4b9b3a5b824ea255e1ea17c522f68d01cd0967 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 19 Mar 2024 16:08:20 +0100 Subject: [PATCH 188/332] Add few notes and a debug check. --- .../AbstractConfigurationComparison.cpp | 29 ++++++++++--------- .../Config/AbstractConfigurationComparison.h | 12 ++++---- src/Common/Config/ConfigHelper.h | 2 ++ .../ExternalDictionariesLoader.cpp | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index e983bd32f51..e241a540cc1 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -37,6 +37,12 @@ namespace { std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); }); + +#if defined(ABORT_ON_LOGICAL_ERROR) + /// Compound `ignore_keys` are not yet implemented. + for (const auto & ignore_key : *ignore_keys) + chassert(ignore_key.find(".") == std::string_view::npos); +#endif } /// Check that the right configuration has the same set of subkeys as the left configuration. @@ -63,7 +69,7 @@ namespace if (!left_subkeys_set.contains(subkey)) return false; - if (!isSameConfiguration(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey))) + if (!isSameConfigurationImpl(left, concatKeyAndSubKey(left_key, subkey), right, concatKeyAndSubKey(right_key, subkey), nullptr)) return false; } return true; @@ -82,6 +88,14 @@ bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const P return isSameConfiguration(left, key, right, key); } +bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys) +{ + const auto * ignore_keys_ptr = !ignore_keys.empty() ? &ignore_keys : nullptr; + return isSameConfigurationImpl(left, left_key, right, right_key, ignore_keys_ptr); +} + bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right, const String & root, const String & name) { if (&left == &right) @@ -99,17 +113,4 @@ bool isSameConfigurationWithMultipleKeys(const Poco::Util::AbstractConfiguration return true; } -bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key) -{ - return isSameConfigurationImpl(left, left_key, right, right_key, /* ignore_keys= */ nullptr); -} - -bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key, - const std::unordered_set & ignore_keys) -{ - return isSameConfigurationImpl(left, left_key, right, right_key, !ignore_keys.empty() ? &ignore_keys : nullptr); -} - } diff --git a/src/Common/Config/AbstractConfigurationComparison.h b/src/Common/Config/AbstractConfigurationComparison.h index edaff8d5363..e8f65a4afcc 100644 --- a/src/Common/Config/AbstractConfigurationComparison.h +++ b/src/Common/Config/AbstractConfigurationComparison.h @@ -11,6 +11,8 @@ namespace Poco::Util namespace DB { /// Returns true if two configurations contains the same keys and values. + /// NOTE: These functions assume no configuration has items having both children and a value + /// (i.e. items like "value"). bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right); @@ -31,13 +33,11 @@ namespace DB const String & key); /// Returns true if specified subviews of the two configurations contains the same keys and values. + /// If `ignore_keys` is specified then the function skips those keys while comparing + /// (even if their values differ, they're considered to be the same.) bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key); - - /// Returns true if specified subviews of the two configurations contains the same keys and values, but without checking specified keys. - bool isSameConfigurationIgnoringKeys(const Poco::Util::AbstractConfiguration & left, const String & left_key, - const Poco::Util::AbstractConfiguration & right, const String & right_key, - const std::unordered_set & ignore_keys); + const Poco::Util::AbstractConfiguration & right, const String & right_key, + const std::unordered_set & ignore_keys = {}); inline bool operator==(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right) { diff --git a/src/Common/Config/ConfigHelper.h b/src/Common/Config/ConfigHelper.h index 4f13bc4ad78..513438bd859 100644 --- a/src/Common/Config/ConfigHelper.h +++ b/src/Common/Config/ConfigHelper.h @@ -14,6 +14,8 @@ namespace DB::ConfigHelper { /// Clones a configuration. +/// NOTE: This function assumes the source configuration doesn't have items having both children and a value +/// (i.e. items like "value"). Poco::AutoPtr clone(const Poco::Util::AbstractConfiguration & src); /// The behavior is like `config.getBool(key, default_)`, diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 34ad240d089..f48ee61dab8 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -55,7 +55,7 @@ bool ExternalDictionariesLoader::doesConfigChangeRequiresReloadingObject(const P ignore_keys.insert("database"); } - return !isSameConfigurationIgnoringKeys(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); + return !isSameConfiguration(old_config, old_key_in_config, new_config, new_key_in_config, ignore_keys); } void ExternalDictionariesLoader::updateObjectFromConfigWithoutReloading(IExternalLoadable & object, const Poco::Util::AbstractConfiguration & config, const String & key_in_config) const From 13ff65474ace15bcd7fc907242a5f7066dacec79 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 19 Mar 2024 21:09:03 +0000 Subject: [PATCH 189/332] impl --- src/Storages/System/StorageSystemReplicas.cpp | 6 +++++- .../02908_many_requests_to_system_replicas.reference | 1 + .../02908_many_requests_to_system_replicas.sh | 11 ++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index af8d67cbc21..35550de11cb 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -135,8 +135,12 @@ public: auto & [_, storage, promise, with_zk_fields] = req; - auto get_status_task = [this, storage, with_zk_fields, promise] () mutable + auto get_status_task = [this, storage, with_zk_fields, promise, thread_group = CurrentThread::getGroup()]() mutable { + SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached();); + if (thread_group) + CurrentThread::attachToGroupIfDetached(thread_group); + try { ReplicatedTableStatus status; diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index f1ca07ef408..17c94686470 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -3,3 +3,4 @@ Making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 900 +1 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 2c57545e603..17e1d87963a 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -49,7 +49,7 @@ echo "Making $CONCURRENCY requests to system.replicas" for i in $(seq 1 $CONCURRENCY) do - curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" 2>&1 || echo "query $i failed" & + curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null SETTINGS log_comment='02908_many_requests';" &>/dev/null & done echo "Query system.replicas while waiting for other concurrent requests to finish" @@ -59,3 +59,12 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(lost_part curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();" 2>&1; wait; + +$CLICKHOUSE_CLIENT -nq " +SYSTEM FLUSH LOGS; + +-- without optimisation there are ~350K zk requests +SELECT sum(ProfileEvents['ZooKeeperTransactions']) < 30000 + FROM system.query_log + WHERE current_database=currentDatabase() AND log_comment='02908_many_requests'; +" From 82b18670a2def4a853fc612344a948b584f59a61 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 19 Mar 2024 22:31:26 +0100 Subject: [PATCH 190/332] Fix error --- src/Analyzer/ColumnTransformers.cpp | 6 +++--- src/Analyzer/ColumnTransformers.h | 13 ++++++------- src/Analyzer/MatcherNode.cpp | 11 +++++++---- src/Parsers/ASTColumnsTransformers.cpp | 3 +++ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Analyzer/ColumnTransformers.cpp b/src/Analyzer/ColumnTransformers.cpp index 40e1e019d50..3a6b9e3b291 100644 --- a/src/Analyzer/ColumnTransformers.cpp +++ b/src/Analyzer/ColumnTransformers.cpp @@ -1,18 +1,18 @@ #include #include +#include #include -#include #include #include -#include #include #include #include + namespace DB { @@ -133,7 +133,7 @@ ExceptColumnTransformerNode::ExceptColumnTransformerNode(std::shared_ptr #include #include -#include + + +namespace re2 +{ + class RE2; +} namespace DB { @@ -102,8 +107,6 @@ enum class ApplyColumnTransformerType /// Get apply column transformer type name const char * toString(ApplyColumnTransformerType type); -class ApplyColumnTransformerNode; -using ApplyColumnTransformerNodePtr = std::shared_ptr; /// Apply column transformer class ApplyColumnTransformerNode final : public IColumnTransformerNode @@ -158,8 +161,6 @@ enum class ExceptColumnTransformerType const char * toString(ExceptColumnTransformerType type); -class ExceptColumnTransformerNode; -using ExceptColumnTransformerNodePtr = std::shared_ptr; /** Except column transformer. * Strict EXCEPT column transformer must use all column names during matched nodes transformation. @@ -230,8 +231,6 @@ private: static constexpr size_t children_size = 0; }; -class ReplaceColumnTransformerNode; -using ReplaceColumnTransformerNodePtr = std::shared_ptr; /** Replace column transformer. * Strict replace column transformer must use all replacements during matched nodes transformation. diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index d0efb0646b3..05861bcc039 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -99,10 +99,13 @@ MatcherNode::MatcherNode(MatcherNodeType matcher_type_, , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) { - columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); - if (!columns_matcher->ok()) - throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + if (!pattern_.empty()) + { + columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + if (!columns_matcher->ok()) + throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + } auto column_transformers_list_node = std::make_shared(); diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 7f3fda9914c..42c4c6ad88a 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -278,6 +278,9 @@ void ASTColumnsExceptTransformer::setPattern(String pattern_) std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { + if (pattern.empty()) + return {}; + auto regexp = std::make_shared(pattern, re2::RE2::Quiet); if (!regexp->ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, From 7275caf5bb3f0f555bbf2155c61822d5f6cdb0de Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 Mar 2024 22:40:00 +0100 Subject: [PATCH 191/332] Update Context.cpp --- src/Interpreters/Context.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f388cf6b3de..b3f16a9ba52 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -566,7 +566,8 @@ struct ContextSharedPart : boost::noncopyable std::lock_guard lock(mutex); delete_async_insert_queue = std::move(async_insert_queue); } - delete_async_insert_queue->flushAndShutdown(); + if (delete_async_insert_queue) + delete_async_insert_queue->flushAndShutdown(); /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config From 6eb488fe517df53aee24bc40dcc2f66a12052ffc Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Mar 2024 23:29:09 +0100 Subject: [PATCH 192/332] Ping CI From cfab620f49206bc3672e57813400857ff5541c7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 01:11:53 +0100 Subject: [PATCH 193/332] Fix bad code --- src/Analyzer/MatcherNode.cpp | 6 +++--- src/Analyzer/MatcherNode.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 05861bcc039..48b60423688 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -53,7 +53,7 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, ColumnTransformersNod { } -MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(String pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, {} /*qualified_identifier*/, {} /*columns_identifiers*/, @@ -62,7 +62,7 @@ MatcherNode::MatcherNode(const String & pattern_, ColumnTransformersNodes column { } -MatcherNode::MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_) +MatcherNode::MatcherNode(Identifier qualified_identifier_, String pattern_, ColumnTransformersNodes column_transformers_) : MatcherNode(MatcherNodeType::COLUMNS_REGEXP, std::move(qualified_identifier_), {} /*columns_identifiers*/, @@ -92,7 +92,7 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - const String & pattern_, + String pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 2217fde6eff..9f6b7c6ce3d 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -59,10 +59,10 @@ public: explicit MatcherNode(Identifier qualified_identifier_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS('regexp') - explicit MatcherNode(const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(String pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant qualified COLUMNS('regexp') - explicit MatcherNode(Identifier qualified_identifier_, const String & pattern_, ColumnTransformersNodes column_transformers_ = {}); + explicit MatcherNode(Identifier qualified_identifier_, String pattern_, ColumnTransformersNodes column_transformers_ = {}); /// Variant unqualified COLUMNS(column_name_1, ...) explicit MatcherNode(Identifiers columns_identifiers_, ColumnTransformersNodes column_transformers_ = {}); @@ -147,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - const String & pattern_, + String pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; From c63a5566dc8f54ed28f07ddf7097bbfeb7f8b0a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 01:25:46 +0100 Subject: [PATCH 194/332] Add a test --- .../03014_invalid_utf8_client.expect | 27 +++++++++++++++++++ .../03014_invalid_utf8_client.reference | 0 2 files changed, 27 insertions(+) create mode 100755 tests/queries/0_stateless/03014_invalid_utf8_client.expect create mode 100644 tests/queries/0_stateless/03014_invalid_utf8_client.reference diff --git a/tests/queries/0_stateless/03014_invalid_utf8_client.expect b/tests/queries/0_stateless/03014_invalid_utf8_client.expect new file mode 100755 index 00000000000..13cbfe408d4 --- /dev/null +++ b/tests/queries/0_stateless/03014_invalid_utf8_client.expect @@ -0,0 +1,27 @@ +#!/usr/bin/expect -f + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 +set history_file $env(CLICKHOUSE_TMP)/$basename.history + +log_user 0 +set timeout 60 +match_max 100000 + +expect_after { + # Do not ignore eof from expect + -i $any_spawn_id eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + -i $any_spawn_id timeout { exit 1 } +} + +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" +expect ":) " + +send -- "SELECT \x99\r" + +expect "Syntax error" + +send "exit\r" +expect eof diff --git a/tests/queries/0_stateless/03014_invalid_utf8_client.reference b/tests/queries/0_stateless/03014_invalid_utf8_client.reference new file mode 100644 index 00000000000..e69de29bb2d From 96e90438e0a1f47fc706792492cf86fe093d0b26 Mon Sep 17 00:00:00 2001 From: nemonlou Date: Wed, 20 Mar 2024 14:25:16 +0800 Subject: [PATCH 195/332] fix clang-tiny --- src/Processors/Transforms/MergeJoinTransform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index b63598483ef..62361bef5e2 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -305,7 +305,7 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( left_to_right_key_remap[left_idx] = right_idx; } - auto smjPtr = typeid_cast(table_join.get()); + const auto *smjPtr = typeid_cast(table_join.get()); if (smjPtr) { null_direction_hint = smjPtr->getNullDirection(); From 2312eb886f93f126be326abd6af1f5ad433d0f29 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 08:53:26 +0100 Subject: [PATCH 196/332] Fix part log for async insert with deduplication --- .../MergeTree/ReplicatedMergeTreeSink.cpp | 14 +++++++++++ ...14_async_with_dedup_part_log_rmt.reference | 5 ++++ .../03014_async_with_dedup_part_log_rmt.sql | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.reference create mode 100644 tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.sql diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index f67076c741b..b43d47bf5f4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -487,7 +487,14 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFa partition.temp_part.finalize(); auto conflict_block_ids = commitPart(zookeeper, partition.temp_part.part, partition.block_id, delayed_chunk->replicas_num, false).first; if (conflict_block_ids.empty()) + { + auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); + PartLog::addNewPart( + storage.getContext(), + PartLog::PartLogEntry(partition.temp_part.part, partition.elapsed_ns, counters_snapshot), + ExecutionStatus(0)); break; + } storage.async_block_ids_cache.triggerCacheUpdate(); ++retry_times; @@ -495,7 +502,14 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithFa /// partition clean conflict partition.filterBlockDuplicate(conflict_block_ids, false); if (partition.block_with_partition.block.rows() == 0) + { + auto counters_snapshot = std::make_shared(partition.part_counters.getPartiallyAtomicSnapshot()); + PartLog::addNewPart( + storage.getContext(), + PartLog::PartLogEntry(partition.temp_part.part, partition.elapsed_ns, counters_snapshot), + ExecutionStatus(ErrorCodes::INSERT_WAS_DEDUPLICATED)); break; + } partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value); /// partition.temp_part is already finalized, no need to call cancel partition.temp_part = storage.writer.writeTempPart(partition.block_with_partition, metadata_snapshot, context); diff --git a/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.reference b/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.reference new file mode 100644 index 00000000000..335c796a9cf --- /dev/null +++ b/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.reference @@ -0,0 +1,5 @@ +-- Inserted part -- +0 1 +-- Deduplicated part -- +0 1 +389 1 diff --git a/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.sql b/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.sql new file mode 100644 index 00000000000..e14c1635853 --- /dev/null +++ b/tests/queries/0_stateless/03014_async_with_dedup_part_log_rmt.sql @@ -0,0 +1,24 @@ +CREATE TABLE 03014_async_with_dedup_part_log (x UInt64) +ENGINE=ReplicatedMergeTree('/clickhouse/table/{database}/03014_async_with_dedup_part_log', 'r1') ORDER BY tuple(); + +SET async_insert = 1; +SET wait_for_async_insert = 1; +SET async_insert_deduplicate = 1; + +SELECT '-- Inserted part --'; +INSERT INTO 03014_async_with_dedup_part_log VALUES (2); + +SYSTEM FLUSH LOGS; +SELECT error, count() FROM system.part_log +WHERE table = '03014_async_with_dedup_part_log' and database = currentDatabase() +GROUP BY error +ORDER BY error; + +SELECT '-- Deduplicated part --'; +INSERT INTO 03014_async_with_dedup_part_log VALUES (2); + +SYSTEM FLUSH LOGS; +SELECT error, count() FROM system.part_log +WHERE table = '03014_async_with_dedup_part_log' and database = currentDatabase() +GROUP BY error +ORDER BY error; From e7aa23d33236443f4fbe36e695a47c6ff4e857d6 Mon Sep 17 00:00:00 2001 From: YenchangChan Date: Mon, 18 Mar 2024 20:14:46 +0800 Subject: [PATCH 197/332] Remove duplicate code in copyS3File.cpp --- src/IO/S3/copyS3File.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 51518df268c..b780c1fc08f 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -194,11 +194,6 @@ namespace auto outcome = client_ptr->CompleteMultipartUpload(request); - if (blob_storage_log) - blob_storage_log->addEvent(BlobStorageLogElement::EventType::MultiPartUploadComplete, - dest_bucket, dest_key, /* local_path_ */ {}, /* data_size */ 0, - outcome.IsSuccess() ? nullptr : &outcome.GetError()); - if (blob_storage_log) blob_storage_log->addEvent(BlobStorageLogElement::EventType::MultiPartUploadComplete, dest_bucket, dest_key, /* local_path_ */ {}, /* data_size */ 0, From 10bf164aa732adc68d914eb3911f1c94aec77fcf Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 20 Mar 2024 10:47:15 +0100 Subject: [PATCH 198/332] Cleanup #ci_set_analyzer --- .../ClusterProxy/executeQuery.cpp | 1 - src/Planner/PlannerJoinTree.cpp | 7 ------- src/Storages/StorageDistributed.cpp | 2 -- src/Storages/buildQueryTreeForShard.cpp | 20 +++++++++---------- .../01455_opentelemetry_distributed.reference | 4 ++-- ..._parallel_replicas_join_subquery.reference | 2 +- ...02771_parallel_replicas_analyzer.reference | 2 +- 7 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 2348081b56c..07ef7aa6c96 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -127,7 +127,6 @@ ContextMutablePtr updateSettingsForCluster(const Cluster & cluster, /// /// Here we don't try to analyze setting again. In case if query_info->additional_filter_ast is not empty, some filter was applied. /// It's just easier to add this filter for a source table. - LOG_DEBUG(&Poco::Logger::get("updateSettings"), "{}", additional_filter_ast != nullptr); if (additional_filter_ast) { Tuple tuple; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e7afd7a1ea4..8ca8f0f258b 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -1,6 +1,5 @@ #include -#include "Common/logger_useful.h" #include #include @@ -64,7 +63,6 @@ #include #include #include -#include namespace DB { @@ -525,7 +523,6 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, return {}; auto const & storage_id = storage->getStorageID(); - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", storage_id.getFullNameNotQuoted()); ASTPtr additional_filter_ast; for (const auto & additional_filter : additional_filters) @@ -546,8 +543,6 @@ FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage, } } - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filter_ast != nullptr); - if (!additional_filter_ast) return {}; @@ -852,8 +847,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const auto & table_expression_alias = table_expression->getOriginalAlias(); auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context); - if (additional_filters_info.actions) - LOG_DEBUG(&Poco::Logger::get("Planner"), "{}", additional_filters_info.actions->dumpDAG()); add_filter(additional_filters_info, "additional filter"); from_stage = storage->getQueryProcessingStage( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 675e5b1071a..a9b909bc71e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -836,8 +836,6 @@ void StorageDistributed::read( { Block header; - LOG_DEBUG(&Poco::Logger::get("read"), "{}", query_info.additional_filter_ast != nullptr); - SelectQueryInfo modified_query_info = query_info; if (local_context->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index ff88172e7a3..5015e89dff7 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -1,26 +1,26 @@ #include -#include #include +#include #include -#include #include -#include +#include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include "Analyzer/QueryNode.h" namespace DB { diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index b04a3a5ea82..2920b387aa2 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,9 +1,9 @@ ===http=== {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} -{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1` SETTINGS allow_experimental_analyzer = 1","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"2"} diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference index 028cc744170..451f0d6d485 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference @@ -31,7 +31,7 @@ 29 2j&S)ba?XG QuQj 17163829389637435056 3 UlI+1 14144472852965836438 =============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE =============== -0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer = 1 +0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') 0 3 SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0; 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1; diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference index 3b8a394a522..5bf3520ccdb 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference @@ -9,4 +9,4 @@ 7885388429666205427 8124171311239967992 1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n allow_experimental_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n allow_experimental_parallel_reading_from_replicas = 1; -0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` SETTINGS allow_experimental_analyzer = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\', allow_experimental_parallel_reading_from_replicas = 1 +0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` From 2c0612c42522007b493f0838dcde7b2030a2d708 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 20 Mar 2024 09:54:49 +0000 Subject: [PATCH 199/332] Empty-Commit From 25bae3e0a5f1b7cb157eb2bc0aa9984e2e6b5274 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 20 Mar 2024 10:58:43 +0000 Subject: [PATCH 200/332] Verbose error medssage about analyzer_compatibility_join_using_top_level_identifier --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 30 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0414ea93f94..c7362cc4cce 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7409,18 +7409,40 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (!result_left_table_expression) result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope); - /// Here we may try to resolve identifier from projection in case it's not resolved from left table expression - /// and analyzer_compatibility_join_using_top_level_identifier is disabled. - /// For now we do not do this, because not all corner cases are clear. + /** Here we may try to resolve identifier from projection in case it's not resolved from left table expression + * and analyzer_compatibility_join_using_top_level_identifier is disabled. + * For now we do not do this, because not all corner cases are clear. + * But let's at least mention it in error message + */ /// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression) /// result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); if (!result_left_table_expression) + { + String extra_message; + const QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; + if (settings.analyzer_compatibility_join_using_top_level_identifier && query_node) + { + for (const auto & projection_node : query_node->getProjection().getNodes()) + { + if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) + { + extra_message = fmt::format( + " , but alias '{}' is present in SELECT list." + " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", + projection_node->formatASTForErrorMessage()); + break; + } + } + } + throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, - "JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}", + "JOIN {} using identifier '{}' cannot be resolved from left table expression{}. In scope {}", join_node_typed.formatASTForErrorMessage(), identifier_full_name, + extra_message, scope.scope_node->formatASTForErrorMessage()); + } if (result_left_table_expression->getNodeType() != QueryTreeNodeType::COLUMN) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, From 5c1a54f643b9c72bf87ab03358a6244a7adf692b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 15:22:50 +0100 Subject: [PATCH 201/332] Fix `clickhouse-client -s` argument --- programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4203e4738dd..9e90596b4f8 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -933,7 +933,7 @@ void Client::addOptions(OptionsDescription & options_description) ("config,c", po::value(), "config-file path (another shorthand)") ("connection", po::value(), "connection to use (from the client config), by default connection name is hostname") ("secure,s", "Use TLS connection") - ("no-secure,s", "Don't use TLS connection") + ("no-secure", "Don't use TLS connection") ("user,u", po::value()->default_value("default"), "user") ("password", po::value(), "password") ("ask-password", "ask-password") From 55f7595c43f7f694dca65fad2880ba92b6d5de41 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 15:24:55 +0100 Subject: [PATCH 202/332] Test `-s` argument for clickhouse-client --- tests/queries/0_stateless/00505_secure.reference | 1 + tests/queries/0_stateless/00505_secure.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/00505_secure.reference b/tests/queries/0_stateless/00505_secure.reference index 9a8656bf491..73bdbdbafbe 100644 --- a/tests/queries/0_stateless/00505_secure.reference +++ b/tests/queries/0_stateless/00505_secure.reference @@ -1,3 +1,4 @@ +1 2 3 4 diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index b7c12911b90..dcda2c0f2e8 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -7,6 +7,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# Use $CLICKHOUSE_CLIENT_SECURE, but replace `--secure` by `-s` to test it +CLICKHOUSE_CLIENT_S=${CLICKHOUSE_CLIENT_SECURE/ --secure / -s } +$CLICKHOUSE_CLIENT_S -q "SELECT 1;" + $CLICKHOUSE_CLIENT_SECURE -q "SELECT 2;" #disable test From 463b90e058c1c6f0b5cd692e2fd699f5b5a1b3f8 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 18 Mar 2024 16:23:07 +0100 Subject: [PATCH 203/332] Add test for `--no-secure` --- tests/queries/0_stateless/00505_secure.reference | 1 + tests/queries/0_stateless/00505_secure.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/00505_secure.reference b/tests/queries/0_stateless/00505_secure.reference index 73bdbdbafbe..24319e2c136 100644 --- a/tests/queries/0_stateless/00505_secure.reference +++ b/tests/queries/0_stateless/00505_secure.reference @@ -1,3 +1,4 @@ +0 1 2 3 diff --git a/tests/queries/0_stateless/00505_secure.sh b/tests/queries/0_stateless/00505_secure.sh index dcda2c0f2e8..eaa50bce6b1 100755 --- a/tests/queries/0_stateless/00505_secure.sh +++ b/tests/queries/0_stateless/00505_secure.sh @@ -7,6 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +$CLICKHOUSE_CLIENT --no-secure -q "SELECT 0;" + # Use $CLICKHOUSE_CLIENT_SECURE, but replace `--secure` by `-s` to test it CLICKHOUSE_CLIENT_S=${CLICKHOUSE_CLIENT_SECURE/ --secure / -s } $CLICKHOUSE_CLIENT_S -q "SELECT 1;" From 624b3a6dbe198eb3c125fe37593b8c22b82c78f8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 Mar 2024 12:08:46 +0000 Subject: [PATCH 204/332] Fixing 00223_shard_distributed_aggregation_memory_efficient with analyzer. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 17 ++++++++++++++++- src/Functions/randConstant.cpp | 16 +++++++++++----- tests/analyzer_tech_debt.txt | 1 - 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 1da5d86edf3..2a46417ee98 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -799,6 +799,8 @@ struct IdentifierResolveScope /// Node hash to mask id map std::shared_ptr> projection_mask_map; + std::map rand_constant_cache; + [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const { const IdentifierResolveScope * scope_to_check = this; @@ -5534,7 +5536,20 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi if (!function) { - function = FunctionFactory::instance().tryGet(function_name, scope.context); + /// This is a hack to allow a query like `select randConstant(), randConstant(), randConstant()`. + /// Function randConstant() would return the same value for the same arguments (in scope). + if (function_name == "randConstant") + { + auto hash = function_node_ptr->getTreeHash(); + auto & func = scope.rand_constant_cache[hash]; + if (!func) + func = FunctionFactory::instance().tryGet(function_name, scope.context); + + function = func; + } + else + function = FunctionFactory::instance().tryGet(function_name, scope.context); + is_executable_udf = false; } diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index 22ce6d88ea6..5872ab8bb34 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -88,6 +88,14 @@ public: return std::make_unique>(); } + RandomConstantOverloadResolver() + { + typename ColumnVector::Container vec_to(1); + + TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); + value = vec_to[0]; + } + DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override { size_t number_of_arguments = data_types.size(); @@ -105,13 +113,11 @@ public: if (!arguments.empty()) argument_types.emplace_back(arguments.back().type); - typename ColumnVector::Container vec_to(1); - - TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); - ToType value = vec_to[0]; - return std::make_unique>(value, argument_types, return_type); } + +private: + ToType value; }; struct NameRandConstant { static constexpr auto name = "randConstant"; }; diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 7cd73705e77..2d8497234e2 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -1,4 +1,3 @@ -00223_shard_distributed_aggregation_memory_efficient 00725_memory_tracking 01062_pm_all_join_with_block_continuation 01083_expressions_in_engine_arguments From 007775c9826853ddcec24282edbd1e1496911b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 13:09:26 +0100 Subject: [PATCH 205/332] Better fuzzer logs --- docker/test/fuzzer/run-fuzzer.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index c9e1681401d..55f4d379005 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -173,16 +173,15 @@ function fuzz mkdir -p /var/run/clickhouse-server - # NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server - # server.log -> CH logs - # stderr.log -> Process logs (sanitizer) + # server.log -> All server logs, including sanitizer + # stderr.log -> Process logs (sanitizer) only clickhouse-server \ --config-file db/config.xml \ --pid-file /var/run/clickhouse-server/clickhouse-server.pid \ -- --path db \ --logger.console=0 \ - --logger.log=server.log > stderr.log 2>&1 & - server_pid=$! + --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 & + server_pid=$(pidof clickhouse-server) kill -0 $server_pid @@ -310,7 +309,7 @@ quit if [ "$server_died" == 1 ] then # The server has died. - if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log stderr.log > description.txt + if ! rg --text -o 'Received signal.*|Logical error.*|Assertion.*failed|Failed assertion.*|.*runtime error: .*|.*is located.*|(SUMMARY|ERROR): [a-zA-Z]+Sanitizer:.*|.*_LIBCPP_ASSERT.*|.*Child process was terminated by signal 9.*' server.log > description.txt then echo "Lost connection to server. See the logs." > description.txt fi From 15339091626617a224ec8a152696b4c4f8881c3a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 13:22:30 +0100 Subject: [PATCH 206/332] Fix flaky 02122_parallel_formatting_Template --- tests/queries/0_stateless/02122_parallel_formatting_Template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02122_parallel_formatting_Template.sh b/tests/queries/0_stateless/02122_parallel_formatting_Template.sh index c7d2329ec14..86a4c017bbe 100755 --- a/tests/queries/0_stateless/02122_parallel_formatting_Template.sh +++ b/tests/queries/0_stateless/02122_parallel_formatting_Template.sh @@ -21,7 +21,7 @@ $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', t diff $non_parallel_file $parallel_file -echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n${rows_read:Quoted}\n${bytes_read:Quoted}\n' > $resultset_path +echo -ne '{prefix} \n${data}\n $$ suffix $$\n${totals}\n${min}\n${max}\n${rows:Quoted}\n${rows_before_limit:Quoted}\n' > $resultset_path echo "Template-2" $CLICKHOUSE_CLIENT -q "select number as x, number + 1 as y, concat('string: ', toString(number)) as s from numbers(200000) group by number with totals order by number limit 190000 format Template $TEMPLATE_SETTINGS" --output_format_parallel_formatting=0 --extremes=1 > $non_parallel_file From 06d28eebe598e975fa14595d1ad1a5a1af3e1999 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:58:21 +0100 Subject: [PATCH 207/332] Fix test reference --- ...159_parallel_formatting_tsv_and_friends.reference | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference index 91e3af03db8..1131cbb670d 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference @@ -1,12 +1,12 @@ TSV, false -194d5061de4cae59489d989373f8effe - +2cc7bfde1a2855814c6ea2c8181679c8 - TSV, true -194d5061de4cae59489d989373f8effe - +2cc7bfde1a2855814c6ea2c8181679c8 - TSVWithNames, false -a6d327a3611288b3f973d00e6116f16e - +c4cb6f9c0d77cd76f2584279993b4438 - TSVWithNames, true -a6d327a3611288b3f973d00e6116f16e - +c4cb6f9c0d77cd76f2584279993b4438 - TSKV, false -c2e32a21c08aacf60bda21248ce4f73f - +fd9ccbc364c90e1f7682348fe7f11a5a - TSKV, true -c2e32a21c08aacf60bda21248ce4f73f - +fd9ccbc364c90e1f7682348fe7f11a5a - From 2c02476f4aea90906fc117067d4adc627b142cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 14:08:00 +0100 Subject: [PATCH 208/332] Change riscv RLIMIT_CPU --- CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d345705ae53..f646c830daa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,11 +61,16 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # set CPU time limit to 1000 seconds set (RLIMIT_CPU 1000) - # Sanitizers are too heavy. Also RISCV has some extra memory requirements - if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE OR ARCH_RISCV64) + # Sanitizers are too heavy + if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE) set (RLIMIT_DATA 10000000000) # 10G endif() + # For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file + if (ARCH_RISCV64) + set (RLIMIT_CPU 1800) + endif() + set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) endif () From 5ed895dbf529415af2896abfda72f9a096461af9 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:12:33 +0100 Subject: [PATCH 209/332] remove whitespace before coma --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c7362cc4cce..6014d761794 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7428,7 +7428,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) { extra_message = fmt::format( - " , but alias '{}' is present in SELECT list." + ", but alias '{}' is present in SELECT list." " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", projection_node->formatASTForErrorMessage()); break; From 52b2220dc219fa7073e4cdb6ab45b37cad90df76 Mon Sep 17 00:00:00 2001 From: divanik Date: Wed, 20 Mar 2024 13:20:39 +0000 Subject: [PATCH 210/332] Resolved issues --- src/Common/iota.cpp | 10 +++---- src/Common/iota.h | 11 +++---- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 30 ++++++++----------- src/Storages/System/StorageSystemNumbers.cpp | 3 +- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/Common/iota.cpp b/src/Common/iota.cpp index 86c9e04bb06..82fe86618c9 100644 --- a/src/Common/iota.cpp +++ b/src/Common/iota.cpp @@ -37,7 +37,7 @@ MULTITARGET_FUNCTION_AVX2_SSE42( ) template -void iota_with_step(T * begin, size_t count, T first_value, T step) +void iotaWithStep(T * begin, size_t count, T first_value, T step) { #if USE_MULTITARGET_CODE if (isArchSupported(TargetArch::AVX2)) @@ -56,10 +56,10 @@ template void iota(UInt64 * begin, size_t count, UInt64 first_value); template void iota(size_t * begin, size_t count, size_t first_value); #endif -template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); -template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); -template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +template void iotaWithStep(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +template void iotaWithStep(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +template void iotaWithStep(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +template void iotaWithStep(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Common/iota.h b/src/Common/iota.h index 8fa18be9769..7ddf61736b9 100644 --- a/src/Common/iota.h +++ b/src/Common/iota.h @@ -32,12 +32,13 @@ extern template void iota(UInt64 * begin, size_t count, UInt64 first_value); extern template void iota(size_t * begin, size_t count, size_t first_value); #endif -template void iota_with_step(T * begin, size_t count, T first_value, T step); +template +void iotaWithStep(T * begin, size_t count, T first_value, T step); -extern template void iota_with_step(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); -extern template void iota_with_step(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); -extern template void iota_with_step(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); +extern template void iotaWithStep(UInt8 * begin, size_t count, UInt8 first_value, UInt8 step); +extern template void iotaWithStep(UInt32 * begin, size_t count, UInt32 first_value, UInt32 step); +extern template void iotaWithStep(UInt64 * begin, size_t count, UInt64 first_value, UInt64 step); #if defined(OS_DARWIN) -extern template void iota_with_step(size_t * begin, size_t count, size_t first_value, size_t step); +extern template void iotaWithStep(size_t * begin, size_t count, size_t first_value, size_t step); #endif } diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 503dd534d89..7a61d09bdd2 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -12,10 +12,7 @@ #include #include #include - -#include #include "Core/Types.h" -#include "base/Decimal_fwd.h" #include "base/types.h" namespace DB @@ -30,12 +27,12 @@ namespace { template -inline void iota_with_step_optimized(T * begin, size_t count, T first_value, T step) +inline void iotaWithStepOptimized(T * begin, size_t count, T first_value, T step) { if (step == 1) iota(begin, count, first_value); else - iota_with_step(begin, count, first_value, step); + iotaWithStep(begin, count, first_value, step); } class NumbersSource : public ISource @@ -76,7 +73,7 @@ protected: UInt64 * current_end = &vec[real_block_size]; - iota_with_step_optimized(pos, static_cast(current_end - pos), curr, step); + iotaWithStepOptimized(pos, static_cast(current_end - pos), curr, step); next += chunk_step; @@ -102,7 +99,7 @@ struct RangeWithStep using RangesWithStep = std::vector; -std::optional stepped_range_from_range(const Range & r, UInt64 step, UInt64 remainder) +std::optional steppedRangeFromRange(const Range & r, UInt64 step, UInt64 remainder) { if ((r.right.get() == 0) && (!r.right_included)) return std::nullopt; @@ -124,7 +121,7 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st return std::optional{RangeWithStep{begin, step, static_cast(right_edge_included - begin) / step + 1}}; } -[[maybe_unused]] auto sizeOfRanges(const RangesWithStep & rs) +auto sizeOfRanges(const RangesWithStep & rs) { UInt128 total_size{}; for (const RangeWithStep & r : rs) @@ -138,7 +135,7 @@ std::optional stepped_range_from_range(const Range & r, UInt64 st /// Generate numbers according to ranges. /// Numbers generated is ordered in one stream. /// Notice that we will not generate additional numbers out of ranges. -class [[maybe_unused]] NumbersRangedSource : public ISource +class NumbersRangedSource : public ISource { public: /// Represent a position in Ranges list. @@ -269,7 +266,7 @@ protected: auto start_value_64 = static_cast(start_value); auto end_value_64 = static_cast(end_value); auto size = (end_value_64 - start_value_64) / this->step; - iota_with_step_optimized(pos, static_cast(size), start_value_64, step); + iotaWithStepOptimized(pos, static_cast(size), start_value_64, step); pos += size; } }; @@ -278,7 +275,7 @@ protected: { UInt64 start_value = range.left + cursor.offset_in_range * step; /// end_value will never overflow - iota_with_step_optimized(pos, static_cast(need), start_value, step); + iotaWithStepOptimized(pos, static_cast(need), start_value, step); pos += need; provided += need; cursor.offset_in_range += need; @@ -331,7 +328,7 @@ private: namespace { /// Whether we should push limit down to scan. -[[maybe_unused]] bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) +bool shouldPushdownLimit(SelectQueryInfo & query_info, UInt64 limit_length) { const auto & query = query_info.query->as(); /// Just ignore some minor cases, such as: @@ -344,7 +341,7 @@ namespace /// Shrink ranges to size. /// For example: ranges: [1, 5], [8, 100]; size: 7, we will get [1, 5], [8, 9] -[[maybe_unused]] void shrinkRanges(RangesWithStep & ranges, size_t size) +void shrinkRanges(RangesWithStep & ranges, size_t size) { size_t last_range_idx = 0; for (size_t i = 0; i < ranges.size(); i++) @@ -480,8 +477,8 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto intersected_range = table_range->intersectWith(r); if (intersected_range.has_value()) { - auto range_with_step = stepped_range_from_range( - intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); + auto range_with_step + = steppedRangeFromRange(intersected_range.value(), numbers_storage.step, numbers_storage.offset % numbers_storage.step); if (range_with_step.has_value()) intersected_ranges.push_back(*range_with_step); } @@ -496,7 +493,7 @@ Pipe ReadFromSystemNumbersStep::makePipe() auto intersected_range = overflowed_table_range->intersectWith(r); if (intersected_range) { - auto range_with_step = stepped_range_from_range( + auto range_with_step = steppedRangeFromRange( intersected_range.value(), numbers_storage.step, static_cast( @@ -518,7 +515,6 @@ Pipe ReadFromSystemNumbersStep::makePipe() const auto & limit_length = limit_length_and_offset.first; const auto & limit_offset = limit_length_and_offset.second; - /// If intersected ranges is limited or we can pushdown limit. UInt128 total_size = sizeOfRanges(intersected_ranges); UInt128 query_limit = limit_length + limit_offset; diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index 32d16ef38f9..629b11ee7f1 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -11,7 +11,6 @@ #include #include #include -#include namespace DB { @@ -26,6 +25,8 @@ StorageSystemNumbers::StorageSystemNumbers( : IStorage(table_id), multithreaded(multithreaded_), limit(limit_), offset(offset_), column_name(column_name_), step(step_) { StorageInMemoryMetadata storage_metadata; + /// This column doesn't have a comment, because otherwise it will be added to all the tables which were created via + /// CREATE TABLE test as numbers(5) storage_metadata.setColumns(ColumnsDescription({{column_name_, std::make_shared()}})); setInMemoryMetadata(storage_metadata); } From 9bf50ac814f59a1936dbe1b64a717718af1cfea8 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 14:50:26 +0100 Subject: [PATCH 211/332] Fix aggregator when data is empty --- src/Interpreters/Aggregator.cpp | 7 ++++++- .../03015_aggregator_empty_data_multiple_blocks.reference | 0 .../03015_aggregator_empty_data_multiple_blocks.sql | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference create mode 100644 tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 1c0d4b2d671..7f3b961a598 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1867,7 +1867,12 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are if (data.empty()) { auto && out_cols = prepareOutputBlockColumns(params, aggregate_functions, getHeader(final), aggregates_pools, final, rows); - return {finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows)}; + auto finalized_block = finalizeBlock(params, getHeader(final), std::move(out_cols), final, rows); + + if (return_single_block) + return std::move(finalized_block); + + return BlocksList{std::move(finalized_block)}; } ConvertToBlockResVariant res; bool use_compiled_functions = false; diff --git a/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql new file mode 100644 index 00000000000..d2c1816f7e8 --- /dev/null +++ b/tests/queries/0_stateless/03015_aggregator_empty_data_multiple_blocks.sql @@ -0,0 +1,3 @@ +CREATE TABLE 03015_aggregator_empty_data_multiple_blocks (c0 Int32) ENGINE = Memory(); +INSERT INTO 03015_aggregator_empty_data_multiple_blocks SELECT * FROM generateRandom() LIMIT 1000; +SELECT radians(t1.c0) FROM 03015_aggregator_empty_data_multiple_blocks AS t1 RIGHT ANTI JOIN 03015_aggregator_empty_data_multiple_blocks AS right_0 ON t1.c0=right_0.c0 GROUP BY t1.c0; From 981c9e0c12ef7f554f48aa7e8402eb7e15368296 Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:30 +0000 Subject: [PATCH 212/332] Restore poco SUN files These were accidentally deleted as unused in https://github.com/ClickHouse/ClickHouse/commit/44afecf083b2cfa3d77d2e227f --- .../include/Poco/FPEnvironment_SUN.h | 75 ++++++++++ .../poco/Foundation/src/FPEnvironment_SUN.cpp | 139 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_SUN.h create mode 100644 base/poco/Foundation/src/FPEnvironment_SUN.cpp diff --git a/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h new file mode 100644 index 00000000000..7b31307e1ca --- /dev/null +++ b/base/poco/Foundation/include/Poco/FPEnvironment_SUN.h @@ -0,0 +1,75 @@ +// +// FPEnvironment_SUN.h +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Definitions of class FPEnvironmentImpl for Solaris. +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Foundation_FPEnvironment_SUN_INCLUDED +#define Foundation_FPEnvironment_SUN_INCLUDED + + +#include +#include "Poco/Foundation.h" + + +namespace Poco +{ + + +class FPEnvironmentImpl +{ +protected: + enum RoundingModeImpl + { + FP_ROUND_DOWNWARD_IMPL = FP_RM, + FP_ROUND_UPWARD_IMPL = FP_RP, + FP_ROUND_TONEAREST_IMPL = FP_RN, + FP_ROUND_TOWARDZERO_IMPL = FP_RZ + }; + enum FlagImpl + { + FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ, + FP_INEXACT_IMPL = FP_X_IMP, + FP_OVERFLOW_IMPL = FP_X_OFL, + FP_UNDERFLOW_IMPL = FP_X_UFL, + FP_INVALID_IMPL = FP_X_INV + }; + FPEnvironmentImpl(); + FPEnvironmentImpl(const FPEnvironmentImpl & env); + ~FPEnvironmentImpl(); + FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env); + void keepCurrentImpl(); + static void clearFlagsImpl(); + static bool isFlagImpl(FlagImpl flag); + static void setRoundingModeImpl(RoundingModeImpl mode); + static RoundingModeImpl getRoundingModeImpl(); + static bool isInfiniteImpl(float value); + static bool isInfiniteImpl(double value); + static bool isInfiniteImpl(long double value); + static bool isNaNImpl(float value); + static bool isNaNImpl(double value); + static bool isNaNImpl(long double value); + static float copySignImpl(float target, float source); + static double copySignImpl(double target, double source); + static long double copySignImpl(long double target, long double source); + +private: + fp_rnd _rnd; + fp_except _exc; +}; + + +} // namespace Poco + + +#endif // Foundation_FPEnvironment_SUN_INCLUDED diff --git a/base/poco/Foundation/src/FPEnvironment_SUN.cpp b/base/poco/Foundation/src/FPEnvironment_SUN.cpp new file mode 100644 index 00000000000..36ee36431df --- /dev/null +++ b/base/poco/Foundation/src/FPEnvironment_SUN.cpp @@ -0,0 +1,139 @@ +// +// FPEnvironment_SUN.cpp +// +// Library: Foundation +// Package: Core +// Module: FPEnvironment +// +// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include +#include "Poco/FPEnvironment_SUN.h" + + +namespace Poco { + + +FPEnvironmentImpl::FPEnvironmentImpl() +{ + _rnd = fpgetround(); + _exc = fpgetmask(); +} + + +FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; +} + + +FPEnvironmentImpl::~FPEnvironmentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env) +{ + _rnd = env._rnd; + _exc = env._exc; + return *this; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(float value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isInfiniteImpl(long double value) +{ + int cls = fpclass(value); + return cls == FP_PINF || cls == FP_NINF; +} + + +bool FPEnvironmentImpl::isNaNImpl(float value) +{ + return isnanf(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(double value) +{ + return isnan(value) != 0; +} + + +bool FPEnvironmentImpl::isNaNImpl(long double value) +{ + return isnan((double) value) != 0; +} + + +float FPEnvironmentImpl::copySignImpl(float target, float source) +{ + return (float) copysign(target, source); +} + + +double FPEnvironmentImpl::copySignImpl(double target, double source) +{ + return (float) copysign(target, source); +} + + +long double FPEnvironmentImpl::copySignImpl(long double target, long double source) +{ + return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target; +} + + +void FPEnvironmentImpl::keepCurrentImpl() +{ + fpsetround(_rnd); + fpsetmask(_exc); +} + + +void FPEnvironmentImpl::clearFlagsImpl() +{ + fpsetsticky(0); +} + + +bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag) +{ + return (fpgetsticky() & flag) != 0; +} + + +void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode) +{ + fpsetround((fp_rnd) mode); +} + + +FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl() +{ + return (FPEnvironmentImpl::RoundingModeImpl) fpgetround(); +} + + +} // namespace Poco From da28eaf454e6fca09dcc6c9b7f616f716ce49e02 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 19 Mar 2024 13:08:38 +0000 Subject: [PATCH 213/332] CI: remove compose files from integration test docker #ci_set_integration --- docker/test/integration/runner/Dockerfile | 1 - tests/integration/README.md | 1 - .../compose/docker_compose_azurite.yml | 0 .../compose/docker_compose_cassandra.yml | 0 .../compose/docker_compose_clickhouse.yml | 0 .../compose/docker_compose_coredns.yml | 0 .../compose/docker_compose_dotnet_client.yml | 0 .../integration}/compose/docker_compose_hdfs.yml | 0 .../integration}/compose/docker_compose_hive.yml | 0 .../compose/docker_compose_jdbc_bridge.yml | 0 .../compose/docker_compose_kafka.yml | 0 .../compose/docker_compose_keeper.yml | 0 .../compose/docker_compose_kerberized_hdfs.yml | 0 .../compose/docker_compose_kerberized_kafka.yml | 0 .../compose/docker_compose_kerberos_kdc.yml | 0 .../integration}/compose/docker_compose_ldap.yml | 0 .../compose/docker_compose_minio.yml | 0 .../compose/docker_compose_mongo.yml | 0 .../compose/docker_compose_mongo_secure.yml | 0 .../compose/docker_compose_mysql.yml | 0 .../compose/docker_compose_mysql_8_0.yml | 0 .../compose/docker_compose_mysql_client.yml | 0 .../compose/docker_compose_mysql_cluster.yml | 0 .../docker_compose_mysql_golang_client.yml | 0 .../compose/docker_compose_mysql_java_client.yml | 0 .../compose/docker_compose_mysql_js_client.yml | 0 .../compose/docker_compose_mysql_php_client.yml | 0 .../integration}/compose/docker_compose_nats.yml | 0 .../integration}/compose/docker_compose_net.yml | 0 .../compose/docker_compose_nginx.yml | 0 .../compose/docker_compose_postgres.yml | 0 .../compose/docker_compose_postgres_cluster.yml | 0 .../compose/docker_compose_postgresql.yml | 0 .../docker_compose_postgresql_java_client.yml | 0 .../compose/docker_compose_rabbitmq.yml | 0 .../compose/docker_compose_redis.yml | 0 .../compose/docker_compose_zookeeper.yml | 0 .../compose/docker_compose_zookeeper_secure.yml | 0 tests/integration/helpers/cluster.py | 16 ++-------------- tests/integration/runner | 2 +- 40 files changed, 3 insertions(+), 17 deletions(-) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_azurite.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_cassandra.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_clickhouse.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_coredns.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_dotnet_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_hdfs.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_hive.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_jdbc_bridge.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_kafka.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_keeper.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_kerberized_hdfs.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_kerberized_kafka.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_kerberos_kdc.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_ldap.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_minio.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mongo.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mongo_secure.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_8_0.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_cluster.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_golang_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_java_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_js_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_mysql_php_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_nats.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_net.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_nginx.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_postgres.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_postgres_cluster.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_postgresql.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_postgresql_java_client.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_rabbitmq.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_redis.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_zookeeper.yml (100%) rename {docker/test/integration/runner => tests/integration}/compose/docker_compose_zookeeper_secure.yml (100%) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index b876f7b9635..8297a7100d1 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -126,7 +126,6 @@ RUN set -x \ COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ -COPY compose/ /compose/ COPY misc/ /misc/ diff --git a/tests/integration/README.md b/tests/integration/README.md index ac01c43769e..cde4cb05aec 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -66,7 +66,6 @@ set the following environment variables: Please note that if you use separate build (`ENABLE_CLICKHOUSE_ALL=OFF`), you need to build different components, including but not limited to `ENABLE_CLICKHOUSE_LIBRARY_BRIDGE=ON ENABLE_CLICKHOUSE_ODBC_BRIDGE=ON ENABLE_CLICKHOUSE_KEEPER=ON`. So it is easier to use `ENABLE_CLICKHOUSE_ALL=ON` -For tests that use common docker compose files you may need to set up their path with environment variable: `DOCKER_COMPOSE_DIR=$HOME/ClickHouse/docker/test/integration/runner/compose` ### Running with runner script diff --git a/docker/test/integration/runner/compose/docker_compose_azurite.yml b/tests/integration/compose/docker_compose_azurite.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_azurite.yml rename to tests/integration/compose/docker_compose_azurite.yml diff --git a/docker/test/integration/runner/compose/docker_compose_cassandra.yml b/tests/integration/compose/docker_compose_cassandra.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_cassandra.yml rename to tests/integration/compose/docker_compose_cassandra.yml diff --git a/docker/test/integration/runner/compose/docker_compose_clickhouse.yml b/tests/integration/compose/docker_compose_clickhouse.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_clickhouse.yml rename to tests/integration/compose/docker_compose_clickhouse.yml diff --git a/docker/test/integration/runner/compose/docker_compose_coredns.yml b/tests/integration/compose/docker_compose_coredns.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_coredns.yml rename to tests/integration/compose/docker_compose_coredns.yml diff --git a/docker/test/integration/runner/compose/docker_compose_dotnet_client.yml b/tests/integration/compose/docker_compose_dotnet_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_dotnet_client.yml rename to tests/integration/compose/docker_compose_dotnet_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_hdfs.yml b/tests/integration/compose/docker_compose_hdfs.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_hdfs.yml rename to tests/integration/compose/docker_compose_hdfs.yml diff --git a/docker/test/integration/runner/compose/docker_compose_hive.yml b/tests/integration/compose/docker_compose_hive.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_hive.yml rename to tests/integration/compose/docker_compose_hive.yml diff --git a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml b/tests/integration/compose/docker_compose_jdbc_bridge.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml rename to tests/integration/compose/docker_compose_jdbc_bridge.yml diff --git a/docker/test/integration/runner/compose/docker_compose_kafka.yml b/tests/integration/compose/docker_compose_kafka.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_kafka.yml rename to tests/integration/compose/docker_compose_kafka.yml diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/tests/integration/compose/docker_compose_keeper.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_keeper.yml rename to tests/integration/compose/docker_compose_keeper.yml diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/tests/integration/compose/docker_compose_kerberized_hdfs.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml rename to tests/integration/compose/docker_compose_kerberized_hdfs.yml diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/tests/integration/compose/docker_compose_kerberized_kafka.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml rename to tests/integration/compose/docker_compose_kerberized_kafka.yml diff --git a/docker/test/integration/runner/compose/docker_compose_kerberos_kdc.yml b/tests/integration/compose/docker_compose_kerberos_kdc.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_kerberos_kdc.yml rename to tests/integration/compose/docker_compose_kerberos_kdc.yml diff --git a/docker/test/integration/runner/compose/docker_compose_ldap.yml b/tests/integration/compose/docker_compose_ldap.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_ldap.yml rename to tests/integration/compose/docker_compose_ldap.yml diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_minio.yml rename to tests/integration/compose/docker_compose_minio.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mongo.yml b/tests/integration/compose/docker_compose_mongo.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mongo.yml rename to tests/integration/compose/docker_compose_mongo.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml b/tests/integration/compose/docker_compose_mongo_secure.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mongo_secure.yml rename to tests/integration/compose/docker_compose_mongo_secure.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql.yml b/tests/integration/compose/docker_compose_mysql.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql.yml rename to tests/integration/compose/docker_compose_mysql.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml b/tests/integration/compose/docker_compose_mysql_8_0.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml rename to tests/integration/compose/docker_compose_mysql_8_0.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_client.yml b/tests/integration/compose/docker_compose_mysql_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_client.yml rename to tests/integration/compose/docker_compose_mysql_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml b/tests/integration/compose/docker_compose_mysql_cluster.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml rename to tests/integration/compose/docker_compose_mysql_cluster.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml b/tests/integration/compose/docker_compose_mysql_golang_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml rename to tests/integration/compose/docker_compose_mysql_golang_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml b/tests/integration/compose/docker_compose_mysql_java_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml rename to tests/integration/compose/docker_compose_mysql_java_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml b/tests/integration/compose/docker_compose_mysql_js_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml rename to tests/integration/compose/docker_compose_mysql_js_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml b/tests/integration/compose/docker_compose_mysql_php_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml rename to tests/integration/compose/docker_compose_mysql_php_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_nats.yml b/tests/integration/compose/docker_compose_nats.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_nats.yml rename to tests/integration/compose/docker_compose_nats.yml diff --git a/docker/test/integration/runner/compose/docker_compose_net.yml b/tests/integration/compose/docker_compose_net.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_net.yml rename to tests/integration/compose/docker_compose_net.yml diff --git a/docker/test/integration/runner/compose/docker_compose_nginx.yml b/tests/integration/compose/docker_compose_nginx.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_nginx.yml rename to tests/integration/compose/docker_compose_nginx.yml diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/tests/integration/compose/docker_compose_postgres.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgres.yml rename to tests/integration/compose/docker_compose_postgres.yml diff --git a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml b/tests/integration/compose/docker_compose_postgres_cluster.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml rename to tests/integration/compose/docker_compose_postgres_cluster.yml diff --git a/docker/test/integration/runner/compose/docker_compose_postgresql.yml b/tests/integration/compose/docker_compose_postgresql.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgresql.yml rename to tests/integration/compose/docker_compose_postgresql.yml diff --git a/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml b/tests/integration/compose/docker_compose_postgresql_java_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml rename to tests/integration/compose/docker_compose_postgresql_java_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/tests/integration/compose/docker_compose_rabbitmq.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_rabbitmq.yml rename to tests/integration/compose/docker_compose_rabbitmq.yml diff --git a/docker/test/integration/runner/compose/docker_compose_redis.yml b/tests/integration/compose/docker_compose_redis.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_redis.yml rename to tests/integration/compose/docker_compose_redis.yml diff --git a/docker/test/integration/runner/compose/docker_compose_zookeeper.yml b/tests/integration/compose/docker_compose_zookeeper.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_zookeeper.yml rename to tests/integration/compose/docker_compose_zookeeper.yml diff --git a/docker/test/integration/runner/compose/docker_compose_zookeeper_secure.yml b/tests/integration/compose/docker_compose_zookeeper_secure.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_zookeeper_secure.yml rename to tests/integration/compose/docker_compose_zookeeper_secure.yml diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index b695b493db7..152b0d066ee 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -55,9 +55,7 @@ from .config_cluster import * HELPERS_DIR = p.dirname(__file__) CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") -LOCAL_DOCKER_COMPOSE_DIR = p.join( - CLICKHOUSE_ROOT_DIR, "docker/test/integration/runner/compose/" -) +LOCAL_DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "tests/integration/compose/") DEFAULT_ENV_NAME = ".env" SANITIZER_SIGN = "==================" @@ -186,17 +184,7 @@ def get_library_bridge_path(): def get_docker_compose_path(): - compose_path = os.environ.get("DOCKER_COMPOSE_DIR") - if compose_path is not None: - return os.path.dirname(compose_path) - else: - if os.path.exists(os.path.dirname("/compose/")): - return os.path.dirname("/compose/") # default in docker runner container - else: - logging.debug( - f"Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {LOCAL_DOCKER_COMPOSE_DIR}" - ) - return LOCAL_DOCKER_COMPOSE_DIR + return LOCAL_DOCKER_COMPOSE_DIR def check_kafka_is_available(kafka_id, kafka_port): diff --git a/tests/integration/runner b/tests/integration/runner index f1d5198f545..2834eaf311b 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -415,7 +415,7 @@ if __name__ == "__main__": cmd = cmd_base + " " + args.command cmd_pre_pull = ( - f"{cmd_base} find /compose -name docker_compose_*.yml " + f"{cmd_base} find /ClickHouse/tests/integration/compose -name docker_compose_*.yml " r"-exec docker-compose -f '{}' pull \;" ) From a343406da47f62f015a5b70590fae479a3894625 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 15:33:45 +0100 Subject: [PATCH 214/332] Another fix for SumIfToCountIfPass --- src/Analyzer/Passes/SumIfToCountIfPass.cpp | 10 ++-- ...3010_sum_to_to_count_if_nullable.reference | 46 +++++++++++++++++++ .../03010_sum_to_to_count_if_nullable.sql | 5 +- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index 1c2097e7be9..1a4712aa697 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -97,13 +97,15 @@ public: if (!if_true_condition_constant_node || !if_false_condition_constant_node) return; + if (auto constant_type = if_true_condition_constant_node->getResultType(); !isNativeInteger(constant_type)) + return; + + if (auto constant_type = if_false_condition_constant_node->getResultType(); !isNativeInteger(constant_type)) + return; + const auto & if_true_condition_constant_value_literal = if_true_condition_constant_node->getValue(); const auto & if_false_condition_constant_value_literal = if_false_condition_constant_node->getValue(); - if (!isInt64OrUInt64FieldType(if_true_condition_constant_value_literal.getType()) || - !isInt64OrUInt64FieldType(if_false_condition_constant_value_literal.getType())) - return; - auto if_true_condition_value = if_true_condition_constant_value_literal.get(); auto if_false_condition_value = if_false_condition_constant_value_literal.get(); diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference index 89e5f639c66..d20634acf93 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference @@ -1,5 +1,6 @@ (5,NULL) (5,NULL) +((6150),3) (5,NULL) QUERY id: 0 PROJECTION COLUMNS @@ -66,3 +67,48 @@ QUERY id: 0 ARGUMENTS LIST id: 19, nodes: 1 CONSTANT id: 20, constant_value: UInt64_10, constant_value_type: UInt8 +((6150),3) +QUERY id: 0 + PROJECTION COLUMNS + ((sum(if(equals(modulo(number, 2), 0), toNullable(0), 123))), toUInt8(3)) Tuple(Tuple(Nullable(UInt64)), UInt8) + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: tuple, function_type: ordinary, result_type: Tuple(Tuple(Nullable(UInt64)), UInt8) + ARGUMENTS + LIST id: 3, nodes: 2 + FUNCTION id: 4, function_name: tuple, function_type: ordinary, result_type: Tuple(Nullable(UInt64)) + ARGUMENTS + LIST id: 5, nodes: 1 + FUNCTION id: 6, function_name: sum, function_type: aggregate, nulls_action : IGNORE_NULLS, result_type: Nullable(UInt64) + ARGUMENTS + LIST id: 7, nodes: 1 + FUNCTION id: 8, function_name: if, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 9, nodes: 3 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + FUNCTION id: 12, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 13, nodes: 2 + COLUMN id: 14, column_name: number, result_type: UInt64, source_id: 15 + CONSTANT id: 16, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 17, constant_value: UInt64_0, constant_value_type: UInt8 + CONSTANT id: 18, constant_value: UInt64_0, constant_value_type: Nullable(UInt8) + EXPRESSION + FUNCTION id: 19, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8) + ARGUMENTS + LIST id: 20, nodes: 1 + CONSTANT id: 21, constant_value: UInt64_0, constant_value_type: UInt8 + CONSTANT id: 22, constant_value: UInt64_123, constant_value_type: UInt8 + CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8 + EXPRESSION + FUNCTION id: 24, function_name: toUInt8, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 25, nodes: 1 + CONSTANT id: 26, constant_value: UInt64_3, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 15, alias: __table1, table_function_name: numbers + ARGUMENTS + LIST id: 27, nodes: 1 + CONSTANT id: 28, constant_value: UInt64_100, constant_value_type: UInt8 diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql index b283a69a020..71a175faac8 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql @@ -3,9 +3,12 @@ SET optimize_rewrite_sum_if_to_count_if = 1; SET allow_experimental_analyzer = 0; SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); +SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); SET allow_experimental_analyzer = 1; SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); EXPLAIN QUERY TREE SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10); SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); -EXPLAIN QUERY TREE SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); \ No newline at end of file +EXPLAIN QUERY TREE SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10); +SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); +EXPLAIN QUERY TREE SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100); From dd1fd5ec391f91ba9656589c8af0a98e95ff3edd Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 20 Mar 2024 15:35:39 +0100 Subject: [PATCH 215/332] Ping CI From e6256fa380721a2d60d29658ae0f8977db1a639e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 15:43:09 +0100 Subject: [PATCH 216/332] Fix build --- src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h index 7807f3d8c25..ccc88079daa 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h @@ -15,6 +15,7 @@ struct MergeTreeDataPartChecksums; struct MergeTreeIndexGranularityInfo; class ISerialization; using SerializationPtr = std::shared_ptr; +class SerializationInfoByName; /** * A class which contains all information about a data part that is required From e34c4618a52b37665c04a3f3b4192add46976cf6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 20 Mar 2024 15:56:54 +0100 Subject: [PATCH 217/332] Fix data race during snapshot destructor call --- src/Coordination/KeeperDispatcher.cpp | 8 +- src/Coordination/KeeperSnapshotManager.h | 3 +- src/Coordination/KeeperStateMachine.cpp | 87 ++++++++++--------- src/Coordination/tests/gtest_coordination.cpp | 2 +- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index cd642087130..84e1632e9b1 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -325,12 +325,12 @@ void KeeperDispatcher::snapshotThread() if (!snapshots_queue.pop(task)) break; - if (shutdown_called) - break; - try { - auto snapshot_file_info = task.create_snapshot(std::move(task.snapshot)); + auto snapshot_file_info = task.create_snapshot(std::move(task.snapshot), /*execute_only_cleanup=*/shutdown_called); + + if (shutdown_called) + break; if (snapshot_file_info.path.empty()) continue; diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index ad3bcee028a..8ba0f92a564 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -98,8 +98,7 @@ struct SnapshotFileInfo }; using KeeperStorageSnapshotPtr = std::shared_ptr; -using CreateSnapshotCallback = std::function; - +using CreateSnapshotCallback = std::function; using SnapshotMetaAndStorage = std::pair; diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 0c398a0d549..3dbdb329b93 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -564,63 +564,65 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res } /// create snapshot task for background execution (in snapshot thread) - snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot) + snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot, bool execute_only_cleanup) { nuraft::ptr exception(nullptr); bool ret = true; - try + if (!execute_only_cleanup) { - { /// Read storage data without locks and create snapshot - std::lock_guard lock(snapshots_lock); + try + { + { /// Read storage data without locks and create snapshot + std::lock_guard lock(snapshots_lock); - if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx()) - { - LOG_INFO( - log, - "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already " - "created", - snapshot->snapshot_meta->get_last_log_idx(), - latest_snapshot_meta->get_last_log_idx()); - } - else - { - latest_snapshot_meta = snapshot->snapshot_meta; - /// we rely on the fact that the snapshot disk cannot be changed during runtime - if (isLocalDisk(*keeper_context->getLatestSnapshotDisk())) + if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx()) { - auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot); - latest_snapshot_info = std::move(snapshot_info); - latest_snapshot_buf = nullptr; + LOG_INFO( + log, + "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already " + "created", + snapshot->snapshot_meta->get_last_log_idx(), + latest_snapshot_meta->get_last_log_idx()); } else { - auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot); - auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx()); - latest_snapshot_info = std::move(snapshot_info); - latest_snapshot_buf = std::move(snapshot_buf); - } + latest_snapshot_meta = snapshot->snapshot_meta; + /// we rely on the fact that the snapshot disk cannot be changed during runtime + if (isLocalDisk(*keeper_context->getLatestSnapshotDisk())) + { + auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot); + latest_snapshot_info = std::move(snapshot_info); + latest_snapshot_buf = nullptr; + } + else + { + auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot); + auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx()); + latest_snapshot_info = std::move(snapshot_info); + latest_snapshot_buf = std::move(snapshot_buf); + } - ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); - LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), latest_snapshot_info.path); + ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreations); + LOG_DEBUG(log, "Created persistent snapshot {} with path {}", latest_snapshot_meta->get_last_log_idx(), latest_snapshot_info.path); + } } } - + catch (...) { - /// Destroy snapshot with lock - std::lock_guard lock(storage_and_responses_lock); - LOG_TRACE(log, "Clearing garbage after snapshot"); - /// Turn off "snapshot mode" and clear outdate part of storage state - storage->clearGarbageAfterSnapshot(); - LOG_TRACE(log, "Cleared garbage after snapshot"); - snapshot.reset(); + ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreationsFailed); + LOG_TRACE(log, "Exception happened during snapshot"); + tryLogCurrentException(log); + ret = false; } } - catch (...) { - ProfileEvents::increment(ProfileEvents::KeeperSnapshotCreationsFailed); - LOG_TRACE(log, "Exception happened during snapshot"); - tryLogCurrentException(log); - ret = false; + /// Destroy snapshot with lock + std::lock_guard lock(storage_and_responses_lock); + LOG_TRACE(log, "Clearing garbage after snapshot"); + /// Turn off "snapshot mode" and clear outdate part of storage state + storage->clearGarbageAfterSnapshot(); + LOG_TRACE(log, "Cleared garbage after snapshot"); + snapshot.reset(); } when_done(ret, exception); @@ -628,11 +630,10 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res return ret ? latest_snapshot_info : SnapshotFileInfo{}; }; - if (keeper_context->getServerState() == KeeperContext::Phase::SHUTDOWN) { LOG_INFO(log, "Creating a snapshot during shutdown because 'create_snapshot_on_exit' is enabled."); - auto snapshot_file_info = snapshot_task.create_snapshot(std::move(snapshot_task.snapshot)); + auto snapshot_file_info = snapshot_task.create_snapshot(std::move(snapshot_task.snapshot), /*execute_only_cleanup=*/false); if (!snapshot_file_info.path.empty() && snapshot_manager_s3) { diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 07dfac0670e..d314757efc9 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -1818,7 +1818,7 @@ void testLogAndStateMachine( bool pop_result = snapshots_queue.pop(snapshot_task); EXPECT_TRUE(pop_result); - snapshot_task.create_snapshot(std::move(snapshot_task.snapshot)); + snapshot_task.create_snapshot(std::move(snapshot_task.snapshot), /*execute_only_cleanup=*/false); } if (snapshot_created && changelog.size() > settings->reserved_log_items) From aa1a210d5d793b16d162d0f49fc92f12de0d7e8c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Mar 2024 17:07:32 +0100 Subject: [PATCH 218/332] Fix logging of autoscaling lambda, add test for effective_capacity --- tests/ci/autoscale_runners_lambda/app.py | 26 ++++++++--------- .../test_autoscale.py | 29 ++++++++++++++++++- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py index 6c3d71708e9..520abcb41bb 100644 --- a/tests/ci/autoscale_runners_lambda/app.py +++ b/tests/ci/autoscale_runners_lambda/app.py @@ -120,13 +120,13 @@ def set_capacity( # Let's calculate a new desired capacity # (capacity_deficit + scale_up - 1) // scale_up : will increase min by 1 # if there is any capacity_deficit - desired_capacity = ( + new_capacity = ( asg["DesiredCapacity"] + (capacity_deficit + scale_up - 1) // scale_up ) - desired_capacity = max(desired_capacity, asg["MinSize"]) - desired_capacity = min(desired_capacity, asg["MaxSize"]) + new_capacity = max(new_capacity, asg["MinSize"]) + new_capacity = min(new_capacity, asg["MaxSize"]) # Finally, should the capacity be even changed - stop = stop or asg["DesiredCapacity"] == desired_capacity + stop = stop or asg["DesiredCapacity"] == new_capacity if stop: logging.info( "Do not increase ASG %s capacity, current capacity=%s, effective " @@ -144,9 +144,9 @@ def set_capacity( "The ASG %s capacity will be increased to %s, current capacity=%s, " "effective capacity=%s, maximum capacity=%s, running jobs=%s, queue size=%s", asg["AutoScalingGroupName"], - desired_capacity, - effective_capacity, + new_capacity, asg["DesiredCapacity"], + effective_capacity, asg["MaxSize"], running, queued, @@ -154,16 +154,16 @@ def set_capacity( if not dry_run: client.set_desired_capacity( AutoScalingGroupName=asg["AutoScalingGroupName"], - DesiredCapacity=desired_capacity, + DesiredCapacity=new_capacity, ) return # Now we will calculate if we need to scale down stop = stop or asg["DesiredCapacity"] == asg["MinSize"] - desired_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down) - desired_capacity = max(desired_capacity, asg["MinSize"]) - desired_capacity = min(desired_capacity, asg["MaxSize"]) - stop = stop or asg["DesiredCapacity"] == desired_capacity + new_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down) + new_capacity = max(new_capacity, asg["MinSize"]) + new_capacity = min(new_capacity, asg["MaxSize"]) + stop = stop or asg["DesiredCapacity"] == new_capacity if stop: logging.info( "Do not decrease ASG %s capacity, current capacity=%s, effective " @@ -181,7 +181,7 @@ def set_capacity( "The ASG %s capacity will be decreased to %s, current capacity=%s, effective " "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s", asg["AutoScalingGroupName"], - desired_capacity, + new_capacity, asg["DesiredCapacity"], effective_capacity, asg["MinSize"], @@ -191,7 +191,7 @@ def set_capacity( if not dry_run: client.set_desired_capacity( AutoScalingGroupName=asg["AutoScalingGroupName"], - DesiredCapacity=desired_capacity, + DesiredCapacity=new_capacity, ) diff --git a/tests/ci/autoscale_runners_lambda/test_autoscale.py b/tests/ci/autoscale_runners_lambda/test_autoscale.py index 75f178ac394..3aeab49ffc7 100644 --- a/tests/ci/autoscale_runners_lambda/test_autoscale.py +++ b/tests/ci/autoscale_runners_lambda/test_autoscale.py @@ -35,7 +35,7 @@ class TestSetCapacity(unittest.TestCase): @property def expected_capacity(self) -> int: - """one-time property""" + """a one-time property""" capacity, self._expected_capacity = self._expected_capacity, -1 return capacity @@ -117,6 +117,33 @@ class TestSetCapacity(unittest.TestCase): set_capacity(t.name, t.queues, self.client, False) self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name) + def test_effective_capacity(self): + """Normal cases test increasing w/o considering + effective_capacity much lower than DesiredCapacity""" + test_cases = ( + TestCase( + "desired-overwritten", + 1, + 20, # DesiredCapacity, overwritten by effective_capacity + 50, + [ + Queue("in_progress", 30, "desired-overwritten"), + Queue("queued", 60, "desired-overwritten"), + ], + 40, + ), + ) + for t in test_cases: + self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size) + # we test that effective_capacity is 30 (a half of 60) + data_with_instances = self.client.expected_data + data_with_instances["AutoScalingGroups"][0]["Instances"] = [ + {"HealthStatus": "Healthy" if i % 2 else "Unhealthy"} for i in range(60) + ] + self.client.expected_data = data_with_instances + set_capacity(t.name, t.queues, self.client, False) + self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name) + def test_exceptions(self): test_cases = ( ( From a7ff6d63c397e729919ab9e31d797b91b6ae4708 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 20 Mar 2024 16:48:24 +0000 Subject: [PATCH 219/332] Fix Non-ready set for system.parts. --- .../System/StorageSystemDetachedParts.cpp | 25 ++++++++++-- .../StorageSystemDroppedTablesParts.cpp | 5 ++- .../System/StorageSystemDroppedTablesParts.h | 6 +-- .../System/StorageSystemPartsBase.cpp | 39 ++++++++++++++++--- src/Storages/System/StorageSystemPartsBase.h | 6 +-- .../0_stateless/02841_not_ready_set_bug.sh | 1 + 6 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 1eb79744022..7207e981561 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -4,10 +4,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -320,16 +322,33 @@ protected: std::shared_ptr storage; std::vector columns_mask; - const ActionsDAG::Node * predicate = nullptr; + ActionsDAGPtr filter; const size_t max_block_size; const size_t num_streams; }; void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes) { + filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) - predicate = filter_actions_dag->getOutputs().at(0); + { + const auto * predicate = filter_actions_dag->getOutputs().at(0); + + Block block; + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "database")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "active")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "uuid")); + + filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter, empty_block, context); + } + } } void StorageSystemDetachedParts::read( @@ -358,7 +377,7 @@ void StorageSystemDetachedParts::read( void ReadFromSystemDetachedParts::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto state = std::make_shared(StoragesInfoStream(predicate, context)); + auto state = std::make_shared(StoragesInfoStream(nullptr, filter, context)); Pipe pipe; diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 20baeee1d3b..3301339df76 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -10,7 +10,7 @@ namespace DB { -StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) +StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context) : StoragesInfoStreamBase(context) { /// Will apply WHERE to subset of columns and then add more columns. @@ -73,7 +73,8 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAG::Node * pr if (block_to_filter.rows()) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter) + VirtualColumnUtils::filterBlockWithDAG(filter, block_to_filter, context); rows = block_to_filter.rows(); } diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.h b/src/Storages/System/StorageSystemDroppedTablesParts.h index a44abea7285..dff9e41cce3 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.h +++ b/src/Storages/System/StorageSystemDroppedTablesParts.h @@ -9,7 +9,7 @@ namespace DB class StoragesDroppedInfoStream : public StoragesInfoStreamBase { public: - StoragesDroppedInfoStream(const ActionsDAG::Node * predicate, ContextPtr context); + StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context); protected: bool tryLockTable(StoragesInfo &) override { @@ -30,9 +30,9 @@ public: std::string getName() const override { return "SystemDroppedTablesParts"; } protected: - std::unique_ptr getStoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) override + std::unique_ptr getStoragesInfoStream(const ActionsDAGPtr &, const ActionsDAGPtr & filter, ContextPtr context) override { - return std::make_unique(predicate, context); + return std::make_unique(filter, context); } }; diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6bdfdd357e8..c81c2c18b2e 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -82,7 +82,7 @@ StoragesInfo::getProjectionParts(MergeTreeData::DataPartStateVector & state, boo return data->getProjectionPartsVectorForInternalUsage({State::Active}, &state); } -StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) +StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context) : StoragesInfoStreamBase(context) { /// Will apply WHERE to subset of columns and then add more columns. @@ -114,7 +114,8 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte std::move(database_column_mut), std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter_by_database) + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, block_to_filter, context); rows = block_to_filter.rows(); /// Block contains new columns, update database_column. @@ -193,7 +194,8 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAG::Node * predicate, Conte if (rows) { /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context); + if (filter_by_other_columns) + VirtualColumnUtils::filterBlockWithDAG(filter_by_other_columns, block_to_filter, context); rows = block_to_filter.rows(); } @@ -225,7 +227,8 @@ protected: std::shared_ptr storage; std::vector columns_mask; const bool has_state_column; - const ActionsDAG::Node * predicate = nullptr; + ActionsDAGPtr filter_by_database; + ActionsDAGPtr filter_by_other_columns; }; ReadFromSystemPartsBase::ReadFromSystemPartsBase( @@ -253,7 +256,31 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) { filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) - predicate = filter_actions_dag->getOutputs().at(0); + { + const auto * predicate = filter_actions_dag->getOutputs().at(0); + + Block block; + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "database")); + + filter_by_database = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter_by_database) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); + } + + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "active")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), "uuid")); + + filter_by_other_columns = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); + if (filter_by_other_columns) + { + auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); + VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); + } + } } void StorageSystemPartsBase::read( @@ -287,7 +314,7 @@ void StorageSystemPartsBase::read( void ReadFromSystemPartsBase::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - auto stream = storage->getStoragesInfoStream(predicate, context); + auto stream = storage->getStoragesInfoStream(filter_by_database, filter_by_other_columns, context); auto header = getOutputStream().header; MutableColumns res_columns = header.cloneEmptyColumns(); diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index 10d1a3a2e0e..be945162b39 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -115,7 +115,7 @@ protected: class StoragesInfoStream : public StoragesInfoStreamBase { public: - StoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context); + StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context); }; /** Implements system table 'parts' which allows to get information about data parts for tables of MergeTree family. @@ -145,9 +145,9 @@ protected: StorageSystemPartsBase(const StorageID & table_id_, ColumnsDescription && columns); - virtual std::unique_ptr getStoragesInfoStream(const ActionsDAG::Node * predicate, ContextPtr context) + virtual std::unique_ptr getStoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context) { - return std::make_unique(predicate, context); + return std::make_unique(filter_by_database, filter_by_other_columns, context); } virtual void diff --git a/tests/queries/0_stateless/02841_not_ready_set_bug.sh b/tests/queries/0_stateless/02841_not_ready_set_bug.sh index 3aaffe51578..9b2f3b0698e 100755 --- a/tests/queries/0_stateless/02841_not_ready_set_bug.sh +++ b/tests/queries/0_stateless/02841_not_ready_set_bug.sh @@ -10,3 +10,4 @@ $CLICKHOUSE_CLIENT -q "insert into t1 select number from numbers(10);" $CLICKHOUSE_CLIENT --max_threads=2 --max_result_rows=1 --result_overflow_mode=break -q "with tab as (select min(number) from t1 prewhere number in (select number from view(select number, row_number() OVER (partition by number % 2 ORDER BY number DESC) from numbers_mt(1e4)) where number != 2 order by number)) select number from t1 union all select * from tab;" > /dev/null $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE 1 in (SELECT number from numbers(2)) AND database = currentDatabase() format Null" +$CLICKHOUSE_CLIENT -q "SELECT xor(1, 0) FROM system.parts WHERE 1 IN (SELECT 1) FORMAT Null" From 54fae360f0a43ba644e667cefbcc28a72e0bc4f9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Mar 2024 17:50:40 +0100 Subject: [PATCH 220/332] Disable duplicate-inclides style check --- utils/check-style/check_cpp.sh | 9 +++++---- utils/check-style/process_style_check_result.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/check-style/check_cpp.sh b/utils/check-style/check_cpp.sh index dc1dae70e22..d647f5eefe1 100755 --- a/utils/check-style/check_cpp.sh +++ b/utils/check-style/check_cpp.sh @@ -6,10 +6,11 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_ou start_total=$(date +%s) -start=$(date +%s) -./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt -runtime=$(($(date +%s)-start)) -echo "Check duplicates. Done. $runtime seconds." +# We decided to have the regexp-based check disabled in favor of clang-tidy +# start=$(date +%s) +# ./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt +# runtime=$(($(date +%s)-start)) +# echo "Check duplicates. Done. $runtime seconds." start=$(date +%s) ./check-style -n |& tee /test_output/style_output.txt diff --git a/utils/check-style/process_style_check_result.py b/utils/check-style/process_style_check_result.py index f1ca53eba3b..e603084732d 100755 --- a/utils/check-style/process_style_check_result.py +++ b/utils/check-style/process_style_check_result.py @@ -13,7 +13,7 @@ def process_result(result_folder): description = "" test_results = [] checks = ( - "duplicate includes", + # "duplicate includes", # disabled in favor of clang-tidy "shellcheck", "style", "pylint", From 952b6b85143b89cc507b6f08a46d662a7d9d87a3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 20 Mar 2024 18:00:02 +0100 Subject: [PATCH 221/332] Use the same features for black check --- utils/check-style/check_py.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/check-style/check_py.sh b/utils/check-style/check_py.sh index 13f4e754ed3..2e645d2f19a 100755 --- a/utils/check-style/check_py.sh +++ b/utils/check-style/check_py.sh @@ -4,9 +4,11 @@ cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_ou start_total=`date +%s` -# FIXME: 1 min to wait + head checkout -echo "Check python formatting with black" | ts +start=`date +%s` +echo "Check " | ts ./check-black -n |& tee /test_output/black_output.txt +runtime=$((`date +%s`-start)) +echo "Check python formatting with black. Done. $runtime seconds." start=`date +%s` ./check-pylint -n |& tee /test_output/pylint_output.txt From 35cc3355971a20d86561aec0a40da00e8cdb4001 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 20 Mar 2024 18:04:35 +0100 Subject: [PATCH 222/332] Revert "Un-flake `test_undrop_query`" --- .../settings.md | 4 +- docs/en/sql-reference/statements/undrop.md | 52 ++++++++++++------- src/Interpreters/DatabaseCatalog.cpp | 4 +- src/Interpreters/InterpreterUndropQuery.cpp | 6 +-- tests/integration/test_undrop_query/test.py | 25 +++------ .../0_stateless/02681_undrop_query.sql | 2 +- 6 files changed, 48 insertions(+), 45 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index f20dcb9025e..07c9a2b88ab 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -933,9 +933,9 @@ Hard limit is configured via system tools ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec} -The delay before a table data is dropped in seconds. If the `DROP TABLE` query has a `SYNC` modifier, this setting is ignored. +Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored. -Default value: `480` (8 minutes). +Default value: `480` (8 minute). ## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} diff --git a/docs/en/sql-reference/statements/undrop.md b/docs/en/sql-reference/statements/undrop.md index 4b138bfe679..40ac1ab4f99 100644 --- a/docs/en/sql-reference/statements/undrop.md +++ b/docs/en/sql-reference/statements/undrop.md @@ -13,6 +13,13 @@ a system table called `system.dropped_tables`. If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view. +:::note +UNDROP TABLE is experimental. To use it add this setting: +```sql +set allow_experimental_undrop_table_query = 1; +``` +::: + :::tip Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md) ::: @@ -25,53 +32,60 @@ UNDROP TABLE [db.]name [UUID ''] [ON CLUSTER cluster] **Example** +``` sql +set allow_experimental_undrop_table_query = 1; +``` + ```sql -CREATE TABLE tab +CREATE TABLE undropMe ( `id` UInt8 ) ENGINE = MergeTree -ORDER BY id; - -DROP TABLE tab; - -SELECT * -FROM system.dropped_tables -FORMAT Vertical; +ORDER BY id ``` +```sql +DROP TABLE undropMe +``` +```sql +SELECT * +FROM system.dropped_tables +FORMAT Vertical +``` ```response Row 1: ────── index: 0 database: default -table: tab +table: undropMe uuid: aa696a1a-1d70-4e60-a841-4c80827706cc engine: MergeTree -metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql +metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql table_dropped_time: 2023-04-05 14:12:12 1 row in set. Elapsed: 0.001 sec. ``` - ```sql -UNDROP TABLE tab; - +UNDROP TABLE undropMe +``` +```response +Ok. +``` +```sql SELECT * FROM system.dropped_tables -FORMAT Vertical; - +FORMAT Vertical +``` ```response Ok. 0 rows in set. Elapsed: 0.001 sec. ``` - ```sql -DESCRIBE TABLE tab -FORMAT Vertical; +DESCRIBE TABLE undropMe +FORMAT Vertical ``` - ```response Row 1: ────── diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index c2f2003aabd..a5a523b658b 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1142,7 +1142,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) TableMarkedAsDropped dropped_table; { std::lock_guard lock(tables_marked_dropped_mutex); - auto latest_drop_time = std::numeric_limits::min(); + time_t latest_drop_time = std::numeric_limits::min(); auto it_dropped_table = tables_marked_dropped.end(); for (auto it = tables_marked_dropped.begin(); it != tables_marked_dropped.end(); ++it) { @@ -1167,7 +1167,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) } if (it_dropped_table == tables_marked_dropped.end()) throw Exception(ErrorCodes::UNKNOWN_TABLE, - "Table {} is being dropped, has been dropped, or the database engine does not support UNDROP", + "The drop task of table {} is in progress, has been dropped or the database engine doesn't support it", table_id.getNameForLogs()); latest_metadata_dropped_path = it_dropped_table->metadata_path; String table_metadata_path = getPathForMetadata(it_dropped_table->table_id); diff --git a/src/Interpreters/InterpreterUndropQuery.cpp b/src/Interpreters/InterpreterUndropQuery.cpp index f628a656947..8401c47df6b 100644 --- a/src/Interpreters/InterpreterUndropQuery.cpp +++ b/src/Interpreters/InterpreterUndropQuery.cpp @@ -17,16 +17,14 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) - : WithMutableContext(context_) - , query_ptr(query_ptr_) +InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) { } + BlockIO InterpreterUndropQuery::execute() { getContext()->checkAccess(AccessType::UNDROP_TABLE); - auto & undrop = query_ptr->as(); if (!undrop.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { diff --git a/tests/integration/test_undrop_query/test.py b/tests/integration/test_undrop_query/test.py index d57aa8c2dc7..590a5690e55 100644 --- a/tests/integration/test_undrop_query/test.py +++ b/tests/integration/test_undrop_query/test.py @@ -29,39 +29,30 @@ def test_undrop_drop_and_undrop_loop(started_cluster): logging.info( "random_sec: " + random_sec.__str__() + ", table_uuid: " + table_uuid ) - node.query( - "CREATE TABLE test_undrop_loop" + "create table test_undrop_loop" + count.__str__() + " UUID '" + table_uuid - + "' (id Int32) ENGINE = MergeTree() ORDER BY id;" + + "' (id Int32) Engine=MergeTree() order by id;" ) - - node.query("DROP TABLE test_undrop_loop" + count.__str__() + ";") - + node.query("drop table test_undrop_loop" + count.__str__() + ";") time.sleep(random_sec) - if random_sec >= 5: error = node.query_and_get_error( - "UNDROP TABLE test_undrop_loop" + "undrop table test_undrop_loop" + count.__str__() - + " UUID '" + + " uuid '" + table_uuid + "';" ) assert "UNKNOWN_TABLE" in error - elif random_sec <= 3: - # (*) + else: node.query( - "UNDROP TABLE test_undrop_loop" + "undrop table test_undrop_loop" + count.__str__() - + " UUID '" + + " uuid '" + table_uuid + "';" ) count = count + 1 - else: - pass - # ignore random_sec = 4 to account for communication delay with the database. - # if we don't do that, then the second case (*) may find the table already dropped and receive an unexpected exception from the database (Bug #55167) diff --git a/tests/queries/0_stateless/02681_undrop_query.sql b/tests/queries/0_stateless/02681_undrop_query.sql index d038a383690..66447fc6c44 100644 --- a/tests/queries/0_stateless/02681_undrop_query.sql +++ b/tests/queries/0_stateless/02681_undrop_query.sql @@ -85,5 +85,5 @@ drop table 02681_undrop_multiple; select table from system.dropped_tables where table = '02681_undrop_multiple' limit 1; undrop table 02681_undrop_multiple; select * from 02681_undrop_multiple order by id; -undrop table 02681_undrop_multiple; -- { serverError TABLE_ALREADY_EXISTS } +undrop table 02681_undrop_multiple; -- { serverError 57 } drop table 02681_undrop_multiple sync; From ed9db6d392ed8b68c6d39ba8ffab81f875bc6949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 20 Mar 2024 18:08:58 +0100 Subject: [PATCH 223/332] Add fuzz queries with CUBE and ROLLUP --- ...03014_analyzer_groupby_fuzz_60317.reference | 10 ++++++++++ .../03014_analyzer_groupby_fuzz_60317.sql | 18 ++++++++++++++++++ ...03015_analyzer_groupby_fuzz_60772.reference | 8 ++++++++ .../03015_analyzer_groupby_fuzz_60772.sql | 13 +++++++++++++ ...03017_analyzer_groupby_fuzz_61600.reference | 3 +++ .../03017_analyzer_groupby_fuzz_61600.sql | 11 +++++++++++ 6 files changed, 63 insertions(+) diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference index 4972904f87d..5e56482a470 100644 --- a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.reference @@ -1 +1,11 @@ 30 30 1970-01-01 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 +30 1970-01-01 30 30 1970-01-01 -1980.1 + +30 1970-01-01 30 30 1970-01-01 -1980.1 diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql index 2f54058526e..094614cb78d 100644 --- a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql +++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql @@ -7,3 +7,21 @@ FROM system.one GROUP BY _CAST(30, 'Nullable(UInt8)') SETTINGS allow_experimental_analyzer = 1; + +-- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + toNullable(toNullable(materialize(_CAST(30, 'LowCardinality(UInt8)')))) AS `toNullable(toNullable(materialize(toLowCardinality(30))))`, + _CAST(0, 'Date') AS `makeDate(-1980.1, -1980.1, 10)`, + _CAST(30, 'LowCardinality(UInt8)') AS `toLowCardinality(30)`, + 30 AS `30`, + makeDate(materialize(_CAST(30, 'LowCardinality(UInt8)')), 10, _CAST(30, 'Nullable(UInt8)')) AS `makeDate(materialize(toLowCardinality(30)), 10, toNullable(toNullable(30)))`, + -1980.1 AS `-1980.1` +FROM system.one AS __table1 +GROUP BY + _CAST(30, 'Nullable(UInt8)'), + -1980.1, + materialize(30), + _CAST(30, 'Nullable(UInt8)') +WITH CUBE +WITH TOTALS +SETTINGS allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference index 611407ecd90..5cdf256a24a 100644 --- a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.reference @@ -1 +1,9 @@ %W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 +%W 2018-01-02 22:33:44 1 diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql index c8b4eef50ff..d3bd9ef0ce3 100644 --- a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql +++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql @@ -8,3 +8,16 @@ GROUP BY 'gr', '2018-01-02 22:33:44' SETTINGS allow_experimental_analyzer = 1; + +-- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + toFixedString(toFixedString(toFixedString(toFixedString(toFixedString(toFixedString('%W', 2), 2), 2), toLowCardinality(toLowCardinality(toNullable(2)))), 2), 2), + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + hasSubsequence(toNullable(materialize(toLowCardinality('garbage'))), 'gr') +GROUP BY + '2018-01-02 22:33:44', + toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19), + 'gr', + '2018-01-02 22:33:44' +WITH CUBE +SETTINGS allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference index 68acb650f8e..eaa1097a734 100644 --- a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.reference @@ -1 +1,4 @@ 38 \N +38 \N +38 \N +38 \N diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql index 44b27750c16..53a5cfe9b1a 100644 --- a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql +++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql @@ -12,3 +12,14 @@ GROUP BY toNullable(3), concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) SETTINGS allow_experimental_analyzer = 1; + +-- WITH ROLLUP (note that result is different with the analyzer (analyzer is correct including all combinations) +SELECT + 38, + concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3))) +FROM set_index_not__fuzz_0 +GROUP BY + toNullable(3), + concat(concat(NULLIF(1, 1), toNullable(toNullable(3)))) +WITH ROLLUP +SETTINGS allow_experimental_analyzer = 1; From 4f273c60c3432f1e919bae87188563e2a078cec1 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 09:46:07 +0000 Subject: [PATCH 224/332] CI: disable some kerberos and hdfs integration tests for arm #ci_set_arm #batch_2 #no_merge_commit --- tests/integration/helpers/cluster.py | 7 +++++++ tests/integration/test_kerberos_auth/test.py | 5 ++++- tests/integration/test_storage_kerberized_hdfs/test.py | 9 ++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 152b0d066ee..b3088ba7c81 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1,8 +1,10 @@ import base64 import errno +from functools import cache import http.client import logging import os +import platform import stat import os.path as p import pprint @@ -4743,3 +4745,8 @@ class ClickHouseKiller(object): def __exit__(self, exc_type, exc_val, exc_tb): self.clickhouse_node.start_clickhouse() + + +@cache +def is_arm(): + return any(arch in platform.processor().lower() for arch in ("arm, aarch")) diff --git a/tests/integration/test_kerberos_auth/test.py b/tests/integration/test_kerberos_auth/test.py index a41255cff3e..0025ca7c6b6 100644 --- a/tests/integration/test_kerberos_auth/test.py +++ b/tests/integration/test_kerberos_auth/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) instance1 = cluster.add_instance( @@ -62,10 +62,12 @@ def make_auth(instance): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_with_keytab(kerberos_cluster): assert make_auth(instance1) == "kuser\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_without_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." @@ -73,6 +75,7 @@ def test_kerberos_auth_without_keytab(kerberos_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_bad_path_to_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index 5ac8b4670f9..c33a9614945 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -3,7 +3,7 @@ import pytest import os -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm import subprocess cluster = ClickHouseCluster(__file__) @@ -29,6 +29,7 @@ def started_cluster(): cluster.shutdown() +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -44,6 +45,7 @@ def test_read_table(started_cluster): assert select_read == data +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -59,6 +61,7 @@ def test_read_write_storage(started_cluster): assert select_read == "1\tMark\t72.53\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_write_storage_not_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -76,6 +79,7 @@ def test_write_storage_not_expired(started_cluster): assert select_read == "1\tMark\t72.53\n" +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_two_users(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -98,6 +102,7 @@ def test_two_users(started_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -118,6 +123,7 @@ def test_read_table_expired(started_cluster): started_cluster.unpause_container("hdfskerberos") +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_prohibited(started_cluster): node1.query( "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')" @@ -132,6 +138,7 @@ def test_prohibited(started_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_cache_path(started_cluster): node1.query( "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')" From 7b8c5f35b522c23a9abae735739496622cdef703 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 14:05:06 +0000 Subject: [PATCH 225/332] disable test_strage_hdfs module #ci_set_arm #batch_2 #no_merge_commit --- tests/integration/test_storage_hdfs/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 121263fb622..9dec1954406 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -2,10 +2,13 @@ import os import pytest import time -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from pyhdfs import HdfsClient +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", From 984c7e69a0a58dfe1930a9e0ee18e8f19f52e9d1 Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 15 Mar 2024 15:18:45 +0100 Subject: [PATCH 226/332] disable test_allowed_url_with_config tests with HDFS #no_merge_commit #ci_set_arm --- tests/ci/ci_config.py | 2 +- .../test_allowed_url_from_config/test.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7c213da27ec..d2caf75f4bd 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -380,7 +380,7 @@ upgrade_check_digest = DigestConfig( docker=["clickhouse/upgrade-check"], ) integration_check_digest = DigestConfig( - include_paths=["./tests/ci/integration_test_check.py", "./tests/integration"], + include_paths=["./tests/ci/integration_test_check.py", "./tests/integration/"], exclude_files=[".md"], docker=IMAGES.copy(), ) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 3106cf12702..df8934aa69b 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", main_configs=["configs/config_with_hosts.xml"]) @@ -16,9 +16,11 @@ node5 = cluster.add_instance( "node5", main_configs=["configs/config_without_allowed_hosts.xml"] ) node6 = cluster.add_instance("node6", main_configs=["configs/config_for_remote.xml"]) -node7 = cluster.add_instance( - "node7", main_configs=["configs/config_for_redirect.xml"], with_hdfs=True -) + +if not is_arm(): + node7 = cluster.add_instance( + "node7", main_configs=["configs/config_for_redirect.xml"], with_hdfs=True + ) @pytest.fixture(scope="module") @@ -270,6 +272,7 @@ def test_table_function_remote(start_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_redirect(start_cluster): hdfs_api = start_cluster.hdfs_api @@ -284,6 +287,7 @@ def test_redirect(start_cluster): node7.query("DROP TABLE table_test_7_1") +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_HDFS(start_cluster): assert "not allowed" in node7.query_and_get_error( "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')" @@ -293,6 +297,7 @@ def test_HDFS(start_cluster): ) +@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_schema_inference(start_cluster): error = node7.query_and_get_error("desc url('http://test.com`, 'TSVRaw'')") assert error.find("ReadWriteBufferFromHTTPBase") == -1 From 55737de0e604760fadeacc5aae82b54d69d52df7 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 15:48:53 +0000 Subject: [PATCH 227/332] disable more hdfs and kafka inttests #ci_set_arm --- tests/integration/test_disk_types/test.py | 9 ++++++--- .../test_endpoint_macro_substitution/test.py | 4 +++- tests/integration/test_format_avro_confluent/test.py | 6 +++++- tests/integration/test_jdbc_bridge/test.py | 5 ++++- tests/integration/test_kafka_bad_messages/test.py | 6 +++++- tests/integration/test_keeper_disks/test.py | 6 +++++- tests/integration/test_log_family_hdfs/test.py | 5 ++++- .../test_materialized_mysql_database/test.py | 5 +++++ tests/integration/test_merge_tree_hdfs/test.py | 6 +++++- tests/integration/test_mysql_protocol/test.py | 11 ++++++++++- tests/integration/test_redirect_url_storage/test.py | 7 ++++++- tests/integration/test_storage_kafka/test.py | 4 +++- 12 files changed, 61 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index b9b8ef2010d..3c4169be4de 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -1,14 +1,17 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV disk_types = { "default": "Local", "disk_s3": "S3", - "disk_hdfs": "HDFS", "disk_encrypted": "S3", } +# do not test HDFS on ARM +if not is_arm(): + disk_types["disk_hdfs"] = "HDFS" + @pytest.fixture(scope="module") def cluster(): @@ -18,7 +21,7 @@ def cluster(): "node", main_configs=["configs/storage.xml"], with_minio=True, - with_hdfs=True, + with_hdfs=not is_arm(), ) cluster.start() diff --git a/tests/integration/test_endpoint_macro_substitution/test.py b/tests/integration/test_endpoint_macro_substitution/test.py index ee72fb9b492..22a649e2225 100644 --- a/tests/integration/test_endpoint_macro_substitution/test.py +++ b/tests/integration/test_endpoint_macro_substitution/test.py @@ -1,5 +1,5 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from pyhdfs import HdfsClient @@ -10,6 +10,8 @@ disk_types = { "disk_encrypted": "S3", } +if is_arm(): + pytestmark = pytest.mark.skip @pytest.fixture(scope="module") def cluster(): diff --git a/tests/integration/test_format_avro_confluent/test.py b/tests/integration/test_format_avro_confluent/test.py index 540f90ae05e..ccaaee83514 100644 --- a/tests/integration/test_format_avro_confluent/test.py +++ b/tests/integration/test_format_avro_confluent/test.py @@ -8,9 +8,13 @@ from confluent_kafka.avro.cached_schema_registry_client import ( CachedSchemaRegistryClient, ) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer -from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm from urllib import parse +# Skip on ARM due to Confluent/Kafka +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_jdbc_bridge/test.py b/tests/integration/test_jdbc_bridge/test.py index 0e41cc8c8b7..c4a0a525df3 100644 --- a/tests/integration/test_jdbc_bridge/test.py +++ b/tests/integration/test_jdbc_bridge/test.py @@ -3,7 +3,7 @@ import os.path as p import pytest import uuid -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.test_tools import TSV from string import Template @@ -14,6 +14,9 @@ instance = cluster.add_instance( datasource = "self" records = 1000 +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index 954b6042305..0446ca5cb47 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -2,10 +2,14 @@ import time import logging import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection from kafka.admin import NewTopic +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( "instance", diff --git a/tests/integration/test_keeper_disks/test.py b/tests/integration/test_keeper_disks/test.py index e41837b89b4..0c91aa03419 100644 --- a/tests/integration/test_keeper_disks/test.py +++ b/tests/integration/test_keeper_disks/test.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm import helpers.keeper_utils as keeper_utils from minio.deleteobjects import DeleteObject import os +if is_arm(): + pytestmark = pytest.mark.skip + + CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) node_logs = cluster.add_instance( diff --git a/tests/integration/test_log_family_hdfs/test.py b/tests/integration/test_log_family_hdfs/test.py index e8afe364ec4..6c3d28d2e3c 100644 --- a/tests/integration/test_log_family_hdfs/test.py +++ b/tests/integration/test_log_family_hdfs/test.py @@ -2,10 +2,13 @@ import logging import sys import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from pyhdfs import HdfsClient +if is_arm(): + pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 89c69c42adc..57e496fe737 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -6,6 +6,7 @@ from helpers.cluster import ( ClickHouseCluster, ClickHouseInstance, get_docker_compose_path, + is_arm, ) import logging @@ -13,6 +14,10 @@ from . import materialized_with_ddl DOCKER_COMPOSE_PATH = get_docker_compose_path() +# skip all test on arm due to no arm support in mysql57 +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) mysql_node = None mysql8_node = None diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 95b63a5c8a3..5ca7dc5feb0 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -3,7 +3,7 @@ import time import os import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.utility import generate_values from helpers.wait_for_helpers import wait_for_delete_inactive_parts from helpers.wait_for_helpers import wait_for_delete_empty_parts @@ -16,6 +16,10 @@ CONFIG_PATH = os.path.join( ) +if is_arm(): + pytestmark = pytest.mark.skip + + def create_table(cluster, table_name, additional_settings=None): node = cluster.instances["node"] diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index 7a69d07633c..e641d4c2300 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -12,11 +12,20 @@ from typing import Literal import docker import pymysql.connections import pytest -from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check +from helpers.cluster import ( + ClickHouseCluster, + get_docker_compose_path, + is_arm, + run_and_check, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index 033f02d7bde..ba3fb3e14ab 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -1,10 +1,15 @@ import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager import threading import time +# skip all tests in the module on ARM due to HDFS +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index dea1ea49851..5bdea179449 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -22,7 +22,7 @@ import kafka.errors import pytest from google.protobuf.internal.encoder import _VarintBytes from helpers.client import QueryRuntimeException -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, is_arm from helpers.network import PartitionManager from helpers.test_tools import TSV from kafka import KafkaAdminClient, KafkaProducer, KafkaConsumer, BrokerConnection @@ -40,6 +40,8 @@ from . import kafka_pb2 from . import social_pb2 from . import message_with_repeated_pb2 +if is_arm(): + pytestmark = pytest.mark.skip # TODO: add test for run-time offset update in CH, if we manually update it on Kafka side. # TODO: add test for SELECT LIMIT is working. From 9f3c625ab5034cadb2805bcd2068f4341eab5829 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Wed, 20 Mar 2024 17:23:08 +0000 Subject: [PATCH 228/332] fix mysql client tests to use mysql8 image #ci_set_integration #ci_set_arm --- .../compose/docker_compose_mysql_client.yml | 2 +- .../test_endpoint_macro_substitution/test.py | 1 + tests/integration/test_kerberos_auth/test.py | 7 ++++--- tests/integration/test_mysql_protocol/test.py | 4 ---- tests/integration/test_storage_kerberized_hdfs/test.py | 10 +++------- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/integration/compose/docker_compose_mysql_client.yml b/tests/integration/compose/docker_compose_mysql_client.yml index 5b37b6e6c09..ee590118d4f 100644 --- a/tests/integration/compose/docker_compose_mysql_client.yml +++ b/tests/integration/compose/docker_compose_mysql_client.yml @@ -1,7 +1,7 @@ version: '2.3' services: mysql_client: - image: mysql:5.7 + image: mysql:8.0 restart: always environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 diff --git a/tests/integration/test_endpoint_macro_substitution/test.py b/tests/integration/test_endpoint_macro_substitution/test.py index 22a649e2225..7dc282a980f 100644 --- a/tests/integration/test_endpoint_macro_substitution/test.py +++ b/tests/integration/test_endpoint_macro_substitution/test.py @@ -13,6 +13,7 @@ disk_types = { if is_arm(): pytestmark = pytest.mark.skip + @pytest.fixture(scope="module") def cluster(): try: diff --git a/tests/integration/test_kerberos_auth/test.py b/tests/integration/test_kerberos_auth/test.py index 0025ca7c6b6..2d0dce33c14 100644 --- a/tests/integration/test_kerberos_auth/test.py +++ b/tests/integration/test_kerberos_auth/test.py @@ -1,6 +1,10 @@ import pytest from helpers.cluster import ClickHouseCluster, is_arm +if is_arm(): + pytestmark = pytest.mark.skip + + cluster = ClickHouseCluster(__file__) instance1 = cluster.add_instance( "instance1", @@ -62,12 +66,10 @@ def make_auth(instance): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_with_keytab(kerberos_cluster): assert make_auth(instance1) == "kuser\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_kerberos_auth_without_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." @@ -75,7 +77,6 @@ def test_kerberos_auth_without_keytab(kerberos_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_bad_path_to_keytab(kerberos_cluster): assert ( "DB::Exception: : Authentication failed: password is incorrect, or there is no user with such name." diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index e641d4c2300..094ae7b9fd0 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -15,16 +15,12 @@ import pytest from helpers.cluster import ( ClickHouseCluster, get_docker_compose_path, - is_arm, run_and_check, ) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() -if is_arm(): - pytestmark = pytest.mark.skip - cluster = ClickHouseCluster(__file__) node = cluster.add_instance( diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index c33a9614945..c72152fa376 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -6,6 +6,9 @@ import os from helpers.cluster import ClickHouseCluster, is_arm import subprocess +if is_arm(): + pytestmark = pytest.mark.skip + cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", @@ -29,7 +32,6 @@ def started_cluster(): cluster.shutdown() -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -45,7 +47,6 @@ def test_read_table(started_cluster): assert select_read == data -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -61,7 +62,6 @@ def test_read_write_storage(started_cluster): assert select_read == "1\tMark\t72.53\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_write_storage_not_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -79,7 +79,6 @@ def test_write_storage_not_expired(started_cluster): assert select_read == "1\tMark\t72.53\n" -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_two_users(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -102,7 +101,6 @@ def test_two_users(started_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_read_table_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -123,7 +121,6 @@ def test_read_table_expired(started_cluster): started_cluster.unpause_container("hdfskerberos") -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_prohibited(started_cluster): node1.query( "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')" @@ -138,7 +135,6 @@ def test_prohibited(started_cluster): ) -@pytest.mark.skipif(is_arm(), reason="skip for ARM") def test_cache_path(started_cluster): node1.query( "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')" From f44127ce19f3723eac91fcc795d0adf0adb0b427 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 20 Mar 2024 19:13:51 +0100 Subject: [PATCH 229/332] fix --- src/Databases/DatabaseReplicatedWorker.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 0a6e8f9345e..51065062995 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -77,10 +77,7 @@ void DatabaseReplicatedDDLWorker::initializeReplication() zookeeper->deleteEphemeralNodeIfContentMatches(active_path, active_id); if (active_node_holder) active_node_holder->setAlreadyRemoved(); - zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); active_node_holder.reset(); - active_node_holder_zookeeper = zookeeper; - active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); String log_ptr_str = zookeeper->get(database->replica_path + "/log_ptr"); UInt32 our_log_ptr = parse(log_ptr_str); @@ -129,9 +126,15 @@ void DatabaseReplicatedDDLWorker::initializeReplication() initializeLogPointer(log_entry_name); } - std::lock_guard lock{database->metadata_mutex}; - if (!database->checkDigestValid(context, false)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Inconsistent database metadata after reconnection to ZooKeeper"); + { + std::lock_guard lock{database->metadata_mutex}; + if (!database->checkDigestValid(context, false)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Inconsistent database metadata after reconnection to ZooKeeper"); + } + + zookeeper->create(active_path, active_id, zkutil::CreateMode::Ephemeral); + active_node_holder_zookeeper = zookeeper; + active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); } String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry) From d190ee8113ae6eae7d77850e884fae7e56c65924 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 20 Mar 2024 19:19:30 +0100 Subject: [PATCH 230/332] Update AsynchronousInsertQueue.h --- src/Interpreters/AsynchronousInsertQueue.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 1a4678e3a7d..a8ee064877d 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -63,6 +63,8 @@ public: PushResult pushQueryWithBlock(ASTPtr query, Block block, ContextPtr query_context); size_t getPoolSize() const { return pool_size; } + /// This method should be called manually because it's not flushed automatically in dtor + /// because all tables may be already unloaded when we destroy AsynchronousInsertQueue void flushAndShutdown(); private: From 6f4ce33f027be0600eea4e563ff06a5eac0894f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 20 Mar 2024 22:12:57 +0100 Subject: [PATCH 231/332] Change how fuzzer arguments are parsed --- programs/local/LocalServer.cpp | 12 ------------ src/Client/ClientBase.cpp | 19 +++++++++++++++++-- src/Client/ClientBase.h | 4 +--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f485e3880c1..6f7d3681c46 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -878,11 +878,6 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &) { -#if defined(FUZZING_MODE) - if (argc) - fuzzer_args.push_back(argv[0]); -#endif - for (int arg_num = 1; arg_num < argc; ++arg_num) { std::string_view arg = argv[arg_num]; @@ -918,13 +913,6 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum arg = argv[arg_num]; addMultiquery(arg, common_arguments); } - else if (arg == "--") - { -#if defined(FUZZING_MODE) - fuzzer_args.insert(fuzzer_args.end(), &argv[arg_num + 1], &argv[argc]); - break; -#endif - } else { common_arguments.emplace_back(arg); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index c4cb6e1a5ad..05f583ef438 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -16,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -70,6 +68,7 @@ #include #include +#include #include #include #include @@ -2638,6 +2637,16 @@ ClientBase * app; void ClientBase::runLibFuzzer() { app = this; + std::vector fuzzer_args_holder; + + if (const char * fuzzer_args_env = getenv("FUZZER_ARGS")) // NOLINT(concurrency-mt-unsafe) + boost::split(fuzzer_args_holder, fuzzer_args_env, isWhitespaceASCII, boost::token_compress_on); + + std::vector fuzzer_args; + fuzzer_args.push_back(argv0); + for (auto & arg : fuzzer_args_holder) + fuzzer_args.emplace_back(arg.data()); + int fuzzer_argc = fuzzer_args.size(); char ** fuzzer_argv = fuzzer_args.data(); @@ -2656,6 +2665,8 @@ void ClientBase::runLibFuzzer() return 0; }); } +#else +void ClientBase::runLibFuzzer() {} #endif @@ -2828,6 +2839,8 @@ void ClientBase::init(int argc, char ** argv) Arguments common_arguments = {""}; /// 0th argument is ignored. std::vector hosts_and_ports_arguments; + if (argc) + argv0 = argv[0]; readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments); /// Support for Unicode dashes @@ -2892,6 +2905,8 @@ void ClientBase::init(int argc, char ** argv) ("interactive", "Process queries-file or --query query and start interactive mode") ("pager", po::value(), "Pipe all output into this command (less or similar)") ("max_memory_usage_in_client", po::value(), "Set memory limit in client/local server") + + ("fuzzer-args", po::value(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.") ; addOptions(options_description); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index c0052ade7aa..51dfaebd995 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -78,10 +78,8 @@ protected: void runInteractive(); void runNonInteractive(); -#if defined(FUZZING_MODE) - std::vector fuzzer_args; + char * argv0 = nullptr; void runLibFuzzer(); -#endif virtual bool processWithFuzzing(const String &) { From a31cf43499b1d02827ee3ca08e3fc961c3911eaa Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 20 Mar 2024 23:13:17 +0100 Subject: [PATCH 232/332] Update the dictionary for OSSFuzz (#61672) --- tests/fuzz/README.md | 18 +- tests/fuzz/all.dict | 3455 +++++++++++++++- tests/fuzz/dictionaries/datatypes.dict | 287 +- tests/fuzz/dictionaries/functions.dict | 5264 +++++++++++++++++++----- tests/fuzz/dictionaries/key_words.dict | 426 +- 5 files changed, 8089 insertions(+), 1361 deletions(-) diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 5b864babde3..6b5b161b2d5 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -1,13 +1,23 @@ -The list of functions generated via following query +The list of functions generated via the following query ``` - clickhouse-client -q "select concat('\"', name, '\"') from system.functions union all select concat('\"', alias_to, '\"') from system.functions where alias_to != '' " > functions.dict + clickhouse client -q "SELECT * FROM (SELECT DISTINCT concat('\"', name, '\"') as res FROM system.functions ORDER BY name UNION ALL SELECT concat('\"', a.name, b.name, '\"') as res FROM system.functions as a CROSS JOIN system.aggregate_function_combinators as b WHERE a.is_aggregate = 1) ORDER BY res" > functions.dict ``` -The list of datatypes generated via following query: +The list of datatypes generated via the following query: ``` - clickhouse-client -q "select concat('\"', name, '\"') from system.data_type_families union all select concat('\"', alias_to, '\"') from system.data_type_families where alias_to != '' " > datatypes.dict + clickhouse client -q "SELECT DISTINCT concat('\"', name, '\"') as res FROM system.data_type_families ORDER BY name" > datatypes.dict +``` + +The list of keywords generated via the following query: + +``` + clickhouse client -q "SELECT DISTINCT concat('\"', keyword, '\"') as res FROM system.keywords ORDER BY keyword" > key_words.dict ``` Then merge all dictionaries into one (all.dict) + +``` + cat ./dictionaries/* | sort | uniq > all.dict +``` \ No newline at end of file diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index b38e2c11a1c..f08e319f0d4 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -1,59 +1,333 @@ - "abs" -"accurate_Cast" "accurateCast" -"accurate_CastOrNull" +"accurateCastOrDefault" "accurateCastOrNull" "acos" "acosh" "ADD" "ADD COLUMN" "ADD CONSTRAINT" +"addDate" "addDays" "addHours" "ADD INDEX" +"addInterval" +"addMicroseconds" +"addMilliseconds" "addMinutes" "addMonths" +"addNanoseconds" +"ADD PROJECTION" "addQuarters" "addressToLine" +"addressToLineWithInlines" "addressToSymbol" "addSeconds" +"ADD STATISTIC" +"addTupleOfIntervals" "addWeeks" "addYears" +"ADMIN OPTION FOR" "aes_decrypt_mysql" "aes_encrypt_mysql" "AFTER" +"age" "AggregateFunction" "aggThrow" +"aggThrowArgMax" +"aggThrowArgMin" +"aggThrowArray" +"aggThrowDistinct" +"aggThrowForEach" +"aggThrowIf" +"aggThrowMap" +"aggThrowMerge" +"aggThrowNull" +"aggThrowOrDefault" +"aggThrowOrNull" +"aggThrowResample" +"aggThrowSimpleState" +"aggThrowState" +"ALGORITHM" "ALIAS" "ALL" +"ALLOWED_LATENESS" "alphaTokens" "ALTER" +"ALTER COLUMN" +"ALTER DATABASE" "ALTER LIVE VIEW" +"ALTER POLICY" +"ALTER PROFILE" +"ALTER QUOTA" +"ALTER ROLE" +"ALTER ROW POLICY" +"ALTER SETTINGS PROFILE" "ALTER TABLE" +"ALTER TEMPORARY TABLE" +"ALTER USER" +"analysisOfVariance" +"analysisOfVarianceArgMax" +"analysisOfVarianceArgMin" +"analysisOfVarianceArray" +"analysisOfVarianceDistinct" +"analysisOfVarianceForEach" +"analysisOfVarianceIf" +"analysisOfVarianceMap" +"analysisOfVarianceMerge" +"analysisOfVarianceNull" +"analysisOfVarianceOrDefault" +"analysisOfVarianceOrNull" +"analysisOfVarianceResample" +"analysisOfVarianceSimpleState" +"analysisOfVarianceState" "and" "AND" +"AND STDOUT" +"anova" +"anovaArgMax" +"anovaArgMin" +"anovaArray" +"anovaDistinct" +"anovaForEach" +"anovaIf" +"anovaMap" +"anovaMerge" +"anovaNull" +"anovaOrDefault" +"anovaOrNull" +"anovaResample" +"anovaSimpleState" +"anovaState" "ANTI" "any" "ANY" +"anyArgMax" +"anyArgMin" +"anyArray" +"anyDistinct" +"anyForEach" "anyHeavy" +"anyHeavyArgMax" +"anyHeavyArgMin" +"anyHeavyArray" +"anyHeavyDistinct" +"anyHeavyForEach" +"anyHeavyIf" +"anyHeavyMap" +"anyHeavyMerge" +"anyHeavyNull" +"anyHeavyOrDefault" +"anyHeavyOrNull" +"anyHeavyResample" +"anyHeavySimpleState" +"anyHeavyState" +"anyIf" "anyLast" +"anyLastArgMax" +"anyLastArgMin" +"anyLastArray" +"anyLastDistinct" +"anyLastForEach" +"anyLastIf" +"anyLastMap" +"anyLastMerge" +"anyLastNull" +"anyLastOrDefault" +"anyLastOrNull" +"anyLastResample" +"anyLast_respect_nulls" +"anyLast_respect_nullsArgMax" +"anyLast_respect_nullsArgMin" +"anyLast_respect_nullsArray" +"anyLast_respect_nullsDistinct" +"anyLast_respect_nullsForEach" +"anyLast_respect_nullsIf" +"anyLast_respect_nullsMap" +"anyLast_respect_nullsMerge" +"anyLast_respect_nullsNull" +"anyLast_respect_nullsOrDefault" +"anyLast_respect_nullsOrNull" +"anyLast_respect_nullsResample" +"anyLast_respect_nullsSimpleState" +"anyLast_respect_nullsState" +"anyLastSimpleState" +"anyLastState" +"anyMap" +"anyMerge" +"anyNull" +"anyOrDefault" +"anyOrNull" +"anyResample" +"any_respect_nulls" +"any_respect_nullsArgMax" +"any_respect_nullsArgMin" +"any_respect_nullsArray" +"any_respect_nullsDistinct" +"any_respect_nullsForEach" +"any_respect_nullsIf" +"any_respect_nullsMap" +"any_respect_nullsMerge" +"any_respect_nullsNull" +"any_respect_nullsOrDefault" +"any_respect_nullsOrNull" +"any_respect_nullsResample" +"any_respect_nullsSimpleState" +"any_respect_nullsState" +"anySimpleState" +"anyState" +"any_value" +"any_valueArgMax" +"any_valueArgMin" +"any_valueArray" +"any_valueDistinct" +"any_valueForEach" +"any_valueIf" +"any_valueMap" +"any_valueMerge" +"any_valueNull" +"any_valueOrDefault" +"any_valueOrNull" +"any_valueResample" +"any_value_respect_nulls" +"any_value_respect_nullsArgMax" +"any_value_respect_nullsArgMin" +"any_value_respect_nullsArray" +"any_value_respect_nullsDistinct" +"any_value_respect_nullsForEach" +"any_value_respect_nullsIf" +"any_value_respect_nullsMap" +"any_value_respect_nullsMerge" +"any_value_respect_nullsNull" +"any_value_respect_nullsOrDefault" +"any_value_respect_nullsOrNull" +"any_value_respect_nullsResample" +"any_value_respect_nullsSimpleState" +"any_value_respect_nullsState" +"any_valueSimpleState" +"any_valueState" +"APPEND" "appendTrailingCharIfAbsent" +"APPLY" +"APPLY DELETED MASK" +"approx_top_count" +"approx_top_countArgMax" +"approx_top_countArgMin" +"approx_top_countArray" +"approx_top_countDistinct" +"approx_top_countForEach" +"approx_top_countIf" +"approx_top_countMap" +"approx_top_countMerge" +"approx_top_countNull" +"approx_top_countOrDefault" +"approx_top_countOrNull" +"approx_top_countResample" +"approx_top_countSimpleState" +"approx_top_countState" +"approx_top_k" +"approx_top_kArgMax" +"approx_top_kArgMin" +"approx_top_kArray" +"approx_top_kDistinct" +"approx_top_kForEach" +"approx_top_kIf" +"approx_top_kMap" +"approx_top_kMerge" +"approx_top_kNull" +"approx_top_kOrDefault" +"approx_top_kOrNull" +"approx_top_kResample" +"approx_top_kSimpleState" +"approx_top_kState" +"approx_top_sum" +"approx_top_sumArgMax" +"approx_top_sumArgMin" +"approx_top_sumArray" +"approx_top_sumDistinct" +"approx_top_sumForEach" +"approx_top_sumIf" +"approx_top_sumMap" +"approx_top_sumMerge" +"approx_top_sumNull" +"approx_top_sumOrDefault" +"approx_top_sumOrNull" +"approx_top_sumResample" +"approx_top_sumSimpleState" +"approx_top_sumState" "argMax" +"argMaxArgMax" +"argMaxArgMin" +"argMaxArray" +"argMaxDistinct" +"argMaxForEach" +"argMaxIf" +"argMaxMap" +"argMaxMerge" +"argMaxNull" +"argMaxOrDefault" +"argMaxOrNull" +"argMaxResample" +"argMaxSimpleState" +"argMaxState" "argMin" +"argMinArgMax" +"argMinArgMin" +"argMinArray" +"argMinDistinct" +"argMinForEach" +"argMinIf" +"argMinMap" +"argMinMerge" +"argMinNull" +"argMinOrDefault" +"argMinOrNull" +"argMinResample" +"argMinSimpleState" +"argMinState" "array" "Array" -"ARRAY" +"array_agg" +"array_aggArgMax" +"array_aggArgMin" +"array_aggArray" +"array_aggDistinct" +"array_aggForEach" +"array_aggIf" +"array_aggMap" +"array_aggMerge" +"array_aggNull" +"array_aggOrDefault" +"array_aggOrNull" +"array_aggResample" +"array_aggSimpleState" +"array_aggState" "arrayAll" "arrayAUC" "arrayAvg" "arrayCompact" "arrayConcat" +"array_concat_agg" +"array_concat_aggArgMax" +"array_concat_aggArgMin" +"array_concat_aggArray" +"array_concat_aggDistinct" +"array_concat_aggForEach" +"array_concat_aggIf" +"array_concat_aggMap" +"array_concat_aggMerge" +"array_concat_aggNull" +"array_concat_aggOrDefault" +"array_concat_aggOrNull" +"array_concat_aggResample" +"array_concat_aggSimpleState" +"array_concat_aggState" "arrayCount" "arrayCumSum" "arrayCumSumNonNegative" "arrayDifference" "arrayDistinct" +"arrayDotProduct" "arrayElement" "arrayEnumerate" "arrayEnumerateDense" @@ -65,10 +339,16 @@ "arrayFilter" "arrayFirst" "arrayFirstIndex" +"arrayFirstOrNull" "arrayFlatten" +"arrayFold" "arrayIntersect" +"arrayJaccardIndex" "arrayJoin" "ARRAY JOIN" +"arrayLast" +"arrayLastIndex" +"arrayLastOrNull" "arrayMap" "arrayMax" "arrayMin" @@ -80,6 +360,7 @@ "arrayProduct" "arrayPushBack" "arrayPushFront" +"arrayRandomSample" "arrayReduce" "arrayReduceInRanges" "arrayResize" @@ -87,6 +368,11 @@ "arrayReverseFill" "arrayReverseSort" "arrayReverseSplit" +"arrayRotateLeft" +"arrayRotateRight" +"arrayShiftLeft" +"arrayShiftRight" +"arrayShingles" "arrayShuffle" "arraySlice" "arraySort" @@ -99,9 +385,11 @@ "AS" "ASC" "ASCENDING" +"ascii" "asin" "asinh" "ASOF" +"ASSUME" "assumeNotNull" "AST" "ASYNC" @@ -111,14 +399,59 @@ "ATTACH" "ATTACH PART" "ATTACH PARTITION" +"ATTACH POLICY" +"ATTACH PROFILE" +"ATTACH QUOTA" +"ATTACH ROLE" +"ATTACH ROW POLICY" +"ATTACH SETTINGS PROFILE" +"ATTACH USER" +"AUTO_INCREMENT" "avg" +"avgArgMax" +"avgArgMin" +"avgArray" +"avgDistinct" +"avgForEach" +"avgIf" +"avgMap" +"avgMerge" +"avgNull" +"avgOrDefault" +"avgOrNull" +"avgResample" +"avgSimpleState" +"avgState" "avgWeighted" +"avgWeightedArgMax" +"avgWeightedArgMin" +"avgWeightedArray" +"avgWeightedDistinct" +"avgWeightedForEach" +"avgWeightedIf" +"avgWeightedMap" +"avgWeightedMerge" +"avgWeightedNull" +"avgWeightedOrDefault" +"avgWeightedOrNull" +"avgWeightedResample" +"avgWeightedSimpleState" +"avgWeightedState" +"AZURE" +"BACKUP" +"bagexpansion" "bar" +"base58Decode" +"base58Encode" "base64Decode" "base64Encode" +"base_backup" "basename" -"bayesAB" +"BCRYPT_HASH" +"BCRYPT_PASSWORD" +"BEGIN TRANSACTION" "BETWEEN" +"BIDIRECTIONAL" "BIGINT" "BIGINT SIGNED" "BIGINT UNSIGNED" @@ -126,8 +459,23 @@ "BINARY" "BINARY LARGE OBJECT" "BINARY VARYING" +"BIT" "bitAnd" "BIT_AND" +"BIT_ANDArgMax" +"BIT_ANDArgMin" +"BIT_ANDArray" +"BIT_ANDDistinct" +"BIT_ANDForEach" +"BIT_ANDIf" +"BIT_ANDMap" +"BIT_ANDMerge" +"BIT_ANDNull" +"BIT_ANDOrDefault" +"BIT_ANDOrNull" +"BIT_ANDResample" +"BIT_ANDSimpleState" +"BIT_ANDState" "__bitBoolMaskAnd" "__bitBoolMaskOr" "bitCount" @@ -156,11 +504,26 @@ "bitNot" "bitOr" "BIT_OR" +"BIT_ORArgMax" +"BIT_ORArgMin" +"BIT_ORArray" +"BIT_ORDistinct" +"BIT_ORForEach" +"BIT_ORIf" +"BIT_ORMap" +"BIT_ORMerge" +"BIT_ORNull" +"BIT_OROrDefault" +"BIT_OROrNull" +"BIT_ORResample" +"BIT_ORSimpleState" +"BIT_ORState" "bitPositionsToArray" "bitRotateLeft" "bitRotateRight" "bitShiftLeft" "bitShiftRight" +"bitSlice" "__bitSwapLastTwo" "bitTest" "bitTestAll" @@ -168,19 +531,53 @@ "__bitWrapperFunc" "bitXor" "BIT_XOR" +"BIT_XORArgMax" +"BIT_XORArgMin" +"BIT_XORArray" +"BIT_XORDistinct" +"BIT_XORForEach" +"BIT_XORIf" +"BIT_XORMap" +"BIT_XORMerge" +"BIT_XORNull" +"BIT_XOROrDefault" +"BIT_XOROrNull" +"BIT_XORResample" +"BIT_XORSimpleState" +"BIT_XORState" +"BLAKE3" "BLOB" "blockNumber" "blockSerializedSize" "blockSize" -"BOOL" -"BOOLEAN" +"bool" +"Bool" +"boolean" "BOTH" "boundingRatio" +"boundingRatioArgMax" +"boundingRatioArgMin" +"boundingRatioArray" +"boundingRatioDistinct" +"boundingRatioForEach" +"boundingRatioIf" +"boundingRatioMap" +"boundingRatioMerge" +"boundingRatioNull" +"boundingRatioOrDefault" +"boundingRatioOrNull" +"boundingRatioResample" +"boundingRatioSimpleState" +"boundingRatioState" "buildId" "BY" "BYTE" "BYTEA" +"byteHammingDistance" "byteSize" +"byteSlice" +"byteSwap" +"CASCADE" "CASE" "caseWithExpr" "caseWithExpression" @@ -188,10 +585,28 @@ "caseWithoutExpression" "_CAST" "CAST" +"catboostEvaluate" "categoricalInformationValue" +"categoricalInformationValueArgMax" +"categoricalInformationValueArgMin" +"categoricalInformationValueArray" +"categoricalInformationValueDistinct" +"categoricalInformationValueForEach" +"categoricalInformationValueIf" +"categoricalInformationValueMap" +"categoricalInformationValueMerge" +"categoricalInformationValueNull" +"categoricalInformationValueOrDefault" +"categoricalInformationValueOrNull" +"categoricalInformationValueResample" +"categoricalInformationValueSimpleState" +"categoricalInformationValueState" "cbrt" "ceil" "ceiling" +"CHANGE" +"CHANGEABLE_IN_READONLY" +"CHANGED" "char" "CHAR" "CHARACTER" @@ -202,13 +617,19 @@ "CHAR_LENGTH" "CHAR VARYING" "CHECK" +"CHECK ALL TABLES" "CHECK TABLE" "cityHash64" -"CLEAR" +"CLEANUP" "CLEAR COLUMN" "CLEAR INDEX" +"CLEAR PROJECTION" +"CLEAR STATISTIC" "CLOB" "CLUSTER" +"cluster_host_ids" +"CLUSTERS" +"CN" "coalesce" "CODEC" "COLLATE" @@ -216,52 +637,301 @@ "COLUMNS" "COMMENT" "COMMENT COLUMN" +"COMMIT" +"COMPRESSION" "concat" "concatAssumeInjective" +"concatWithSeparator" +"concatWithSeparatorAssumeInjective" +"concat_ws" "connection_id" -"connectionid" "connectionId" +"CONST" "CONSTRAINT" +"contingency" +"contingencyArgMax" +"contingencyArgMin" +"contingencyArray" +"contingencyDistinct" +"contingencyForEach" +"contingencyIf" +"contingencyMap" +"contingencyMerge" +"contingencyNull" +"contingencyOrDefault" +"contingencyOrNull" +"contingencyResample" +"contingencySimpleState" +"contingencyState" "convertCharset" "corr" +"corrArgMax" +"corrArgMin" +"corrArray" +"corrDistinct" +"corrForEach" +"corrIf" +"corrMap" +"corrMatrix" +"corrMatrixArgMax" +"corrMatrixArgMin" +"corrMatrixArray" +"corrMatrixDistinct" +"corrMatrixForEach" +"corrMatrixIf" +"corrMatrixMap" +"corrMatrixMerge" +"corrMatrixNull" +"corrMatrixOrDefault" +"corrMatrixOrNull" +"corrMatrixResample" +"corrMatrixSimpleState" +"corrMatrixState" +"corrMerge" +"corrNull" +"corrOrDefault" +"corrOrNull" +"corrResample" +"corrSimpleState" "corrStable" +"corrStableArgMax" +"corrStableArgMin" +"corrStableArray" +"corrStableDistinct" +"corrStableForEach" +"corrStableIf" +"corrStableMap" +"corrStableMerge" +"corrStableNull" +"corrStableOrDefault" +"corrStableOrNull" +"corrStableResample" +"corrStableSimpleState" +"corrStableState" +"corrState" "cos" "cosh" +"cosineDistance" "count" +"countArgMax" +"countArgMin" +"countArray" "countDigits" +"countDistinct" "countEqual" +"countForEach" +"countIf" +"countMap" "countMatches" "countMatchesCaseInsensitive" +"countMerge" +"countNull" +"countOrDefault" +"countOrNull" +"countResample" +"countSimpleState" +"countState" "countSubstrings" "countSubstringsCaseInsensitive" "countSubstringsCaseInsensitiveUTF8" "covarPop" "COVAR_POP" +"covarPopArgMax" +"COVAR_POPArgMax" +"covarPopArgMin" +"COVAR_POPArgMin" +"covarPopArray" +"COVAR_POPArray" +"covarPopDistinct" +"COVAR_POPDistinct" +"covarPopForEach" +"COVAR_POPForEach" +"covarPopIf" +"COVAR_POPIf" +"covarPopMap" +"COVAR_POPMap" +"covarPopMatrix" +"covarPopMatrixArgMax" +"covarPopMatrixArgMin" +"covarPopMatrixArray" +"covarPopMatrixDistinct" +"covarPopMatrixForEach" +"covarPopMatrixIf" +"covarPopMatrixMap" +"covarPopMatrixMerge" +"covarPopMatrixNull" +"covarPopMatrixOrDefault" +"covarPopMatrixOrNull" +"covarPopMatrixResample" +"covarPopMatrixSimpleState" +"covarPopMatrixState" +"covarPopMerge" +"COVAR_POPMerge" +"covarPopNull" +"COVAR_POPNull" +"covarPopOrDefault" +"COVAR_POPOrDefault" +"covarPopOrNull" +"COVAR_POPOrNull" +"covarPopResample" +"COVAR_POPResample" +"covarPopSimpleState" +"COVAR_POPSimpleState" "covarPopStable" +"covarPopStableArgMax" +"covarPopStableArgMin" +"covarPopStableArray" +"covarPopStableDistinct" +"covarPopStableForEach" +"covarPopStableIf" +"covarPopStableMap" +"covarPopStableMerge" +"covarPopStableNull" +"covarPopStableOrDefault" +"covarPopStableOrNull" +"covarPopStableResample" +"covarPopStableSimpleState" +"covarPopStableState" +"covarPopState" +"COVAR_POPState" "covarSamp" "COVAR_SAMP" +"covarSampArgMax" +"COVAR_SAMPArgMax" +"covarSampArgMin" +"COVAR_SAMPArgMin" +"covarSampArray" +"COVAR_SAMPArray" +"covarSampDistinct" +"COVAR_SAMPDistinct" +"covarSampForEach" +"COVAR_SAMPForEach" +"covarSampIf" +"COVAR_SAMPIf" +"covarSampMap" +"COVAR_SAMPMap" +"covarSampMatrix" +"covarSampMatrixArgMax" +"covarSampMatrixArgMin" +"covarSampMatrixArray" +"covarSampMatrixDistinct" +"covarSampMatrixForEach" +"covarSampMatrixIf" +"covarSampMatrixMap" +"covarSampMatrixMerge" +"covarSampMatrixNull" +"covarSampMatrixOrDefault" +"covarSampMatrixOrNull" +"covarSampMatrixResample" +"covarSampMatrixSimpleState" +"covarSampMatrixState" +"covarSampMerge" +"COVAR_SAMPMerge" +"covarSampNull" +"COVAR_SAMPNull" +"covarSampOrDefault" +"COVAR_SAMPOrDefault" +"covarSampOrNull" +"COVAR_SAMPOrNull" +"covarSampResample" +"COVAR_SAMPResample" +"covarSampSimpleState" +"COVAR_SAMPSimpleState" "covarSampStable" +"covarSampStableArgMax" +"covarSampStableArgMin" +"covarSampStableArray" +"covarSampStableDistinct" +"covarSampStableForEach" +"covarSampStableIf" +"covarSampStableMap" +"covarSampStableMerge" +"covarSampStableNull" +"covarSampStableOrDefault" +"covarSampStableOrNull" +"covarSampStableResample" +"covarSampStableSimpleState" +"covarSampStableState" +"covarSampState" +"COVAR_SAMPState" +"cramersV" +"cramersVArgMax" +"cramersVArgMin" +"cramersVArray" +"cramersVBiasCorrected" +"cramersVBiasCorrectedArgMax" +"cramersVBiasCorrectedArgMin" +"cramersVBiasCorrectedArray" +"cramersVBiasCorrectedDistinct" +"cramersVBiasCorrectedForEach" +"cramersVBiasCorrectedIf" +"cramersVBiasCorrectedMap" +"cramersVBiasCorrectedMerge" +"cramersVBiasCorrectedNull" +"cramersVBiasCorrectedOrDefault" +"cramersVBiasCorrectedOrNull" +"cramersVBiasCorrectedResample" +"cramersVBiasCorrectedSimpleState" +"cramersVBiasCorrectedState" +"cramersVDistinct" +"cramersVForEach" +"cramersVIf" +"cramersVMap" +"cramersVMerge" +"cramersVNull" +"cramersVOrDefault" +"cramersVOrNull" +"cramersVResample" +"cramersVSimpleState" +"cramersVState" "CRC32" "CRC32IEEE" "CRC64" "CREATE" +"CREATE POLICY" +"CREATE PROFILE" +"CREATE QUOTA" +"CREATE ROLE" +"CREATE ROW POLICY" +"CREATE SETTINGS PROFILE" +"CREATE TABLE" +"CREATE TEMPORARY TABLE" +"CREATE USER" "CROSS" "CUBE" +"curdate" +"current_database" "currentDatabase" +"current_date" +"CURRENT GRANTS" "currentProfiles" +"CURRENT QUOTA" "currentRoles" +"CURRENT ROLES" +"CURRENT ROW" +"current_schemas" +"currentSchemas" +"current_timestamp" +"CURRENT TRANSACTION" "currentUser" +"CURRENT_USER" +"CURRENTUSER" "cutFragment" "cutIPv6" "cutQueryString" "cutQueryStringAndFragment" "cutToFirstSignificantSubdomain" "cutToFirstSignificantSubdomainCustom" +"cutToFirstSignificantSubdomainCustomRFC" "cutToFirstSignificantSubdomainCustomWithWWW" +"cutToFirstSignificantSubdomainCustomWithWWWRFC" +"cutToFirstSignificantSubdomainRFC" "cutToFirstSignificantSubdomainWithWWW" +"cutToFirstSignificantSubdomainWithWWWRFC" "cutURLParameter" "cutWWW" "D" +"damerauLevenshteinDistance" "DATABASE" "DATABASES" "Date" @@ -269,9 +939,11 @@ "Date32" "DATE_ADD" "DATEADD" +"date_diff" "dateDiff" "DATE_DIFF" "DATEDIFF" +"DATE_FORMAT" "dateName" "DATE_SUB" "DATESUB" @@ -280,12 +952,13 @@ "DateTime64" "dateTime64ToSnowflake" "dateTimeToSnowflake" -"date_trunc" "dateTrunc" +"DATE_TRUNC" "DAY" "DAYOFMONTH" "DAYOFWEEK" "DAYOFYEAR" +"DAYS" "DD" "DEC" "Decimal" @@ -293,28 +966,84 @@ "Decimal256" "Decimal32" "Decimal64" +"decodeHTMLComponent" "decodeURLComponent" +"decodeURLFormComponent" "decodeXMLComponent" "decrypt" "DEDUPLICATE" "DEFAULT" +"DEFAULT DATABASE" "defaultProfiles" +"DEFAULT ROLE" "defaultRoles" "defaultValueOfArgumentType" "defaultValueOfTypeName" -"DELAY" +"DEFINER" +"degrees" "DELETE" "DELETE WHERE" "deltaSum" +"deltaSumArgMax" +"deltaSumArgMin" +"deltaSumArray" +"deltaSumDistinct" +"deltaSumForEach" +"deltaSumIf" +"deltaSumMap" +"deltaSumMerge" +"deltaSumNull" +"deltaSumOrDefault" +"deltaSumOrNull" +"deltaSumResample" +"deltaSumSimpleState" +"deltaSumState" "deltaSumTimestamp" +"deltaSumTimestampArgMax" +"deltaSumTimestampArgMin" +"deltaSumTimestampArray" +"deltaSumTimestampDistinct" +"deltaSumTimestampForEach" +"deltaSumTimestampIf" +"deltaSumTimestampMap" +"deltaSumTimestampMerge" +"deltaSumTimestampNull" +"deltaSumTimestampOrDefault" +"deltaSumTimestampOrNull" +"deltaSumTimestampResample" +"deltaSumTimestampSimpleState" +"deltaSumTimestampState" "demangle" "dense_rank" +"dense_rankArgMax" +"dense_rankArgMin" +"dense_rankArray" +"dense_rankDistinct" +"dense_rankForEach" +"dense_rankIf" +"dense_rankMap" +"dense_rankMerge" +"dense_rankNull" +"dense_rankOrDefault" +"dense_rankOrNull" +"dense_rankResample" +"dense_rankSimpleState" +"dense_rankState" +"DEPENDS ON" "DESC" "DESCENDING" "DESCRIBE" "DETACH" +"DETACH PART" "DETACH PARTITION" +"detectCharset" +"detectLanguage" +"detectLanguageMixed" +"detectLanguageUnknown" +"detectProgrammingLanguage" +"detectTonality" "dictGet" +"dictGetAll" "dictGetChildren" "dictGetDate" "dictGetDateOrDefault" @@ -334,6 +1063,10 @@ "dictGetInt64OrDefault" "dictGetInt8" "dictGetInt8OrDefault" +"dictGetIPv4" +"dictGetIPv4OrDefault" +"dictGetIPv6" +"dictGetIPv6OrDefault" "dictGetOrDefault" "dictGetOrNull" "dictGetString" @@ -353,24 +1086,45 @@ "DICTIONARY" "dictIsIn" "DISK" +"displayName" +"distanceL1" +"distanceL2" +"distanceL2Squared" +"distanceLinf" +"distanceLp" "DISTINCT" -"DISTRIBUTED" +"DISTINCT ON" +"DIV" "divide" +"divideDecimal" "domain" +"domainRFC" "domainWithoutWWW" +"domainWithoutWWWRFC" +"dotProduct" "DOUBLE" "DOUBLE PRECISION" +"DOUBLE_SHA1_HASH" +"DOUBLE_SHA1_PASSWORD" "DROP" "DROP COLUMN" "DROP CONSTRAINT" +"DROP DEFAULT" "DROP DETACHED PART" "DROP DETACHED PARTITION" "DROP INDEX" +"DROP PART" "DROP PARTITION" +"DROP PROJECTION" +"DROP STATISTIC" +"DROP TABLE" +"DROP TEMPORARY TABLE" "dumpColumnStructure" "e" +"editDistance" "ELSE" "empty" +"EMPTY" "emptyArrayDate" "emptyArrayDateTime" "emptyArrayFloat32" @@ -385,24 +1139,55 @@ "emptyArrayUInt32" "emptyArrayUInt64" "emptyArrayUInt8" +"EMPTY AS" "enabledProfiles" "enabledRoles" +"ENABLED ROLES" +"encodeURLComponent" +"encodeURLFormComponent" "encodeXMLComponent" "encrypt" "END" "endsWith" +"endsWithUTF8" +"ENFORCED" "ENGINE" "entropy" +"entropyArgMax" +"entropyArgMin" +"entropyArray" +"entropyDistinct" +"entropyForEach" +"entropyIf" +"entropyMap" +"entropyMerge" +"entropyNull" +"entropyOrDefault" +"entropyOrNull" +"entropyResample" +"entropySimpleState" +"entropyState" "Enum" "ENUM" "Enum16" "Enum8" +"EPHEMERAL" +"EPHEMERAL SEQUENTIAL" "equals" "erf" "erfc" "errorCodeToName" +"ESTIMATE" "evalMLMethod" +"EVENT" "EVENTS" +"EVERY" +"EXCEPT" +"EXCEPT DATABASE" +"EXCEPT DATABASES" +"EXCEPT TABLE" +"EXCEPT TABLES" +"EXCHANGE DICTIONARIES" "EXCHANGE TABLES" "EXISTS" "exp" @@ -410,7 +1195,83 @@ "exp2" "EXPLAIN" "exponentialMovingAverage" +"exponentialMovingAverageArgMax" +"exponentialMovingAverageArgMin" +"exponentialMovingAverageArray" +"exponentialMovingAverageDistinct" +"exponentialMovingAverageForEach" +"exponentialMovingAverageIf" +"exponentialMovingAverageMap" +"exponentialMovingAverageMerge" +"exponentialMovingAverageNull" +"exponentialMovingAverageOrDefault" +"exponentialMovingAverageOrNull" +"exponentialMovingAverageResample" +"exponentialMovingAverageSimpleState" +"exponentialMovingAverageState" +"exponentialTimeDecayedAvg" +"exponentialTimeDecayedAvgArgMax" +"exponentialTimeDecayedAvgArgMin" +"exponentialTimeDecayedAvgArray" +"exponentialTimeDecayedAvgDistinct" +"exponentialTimeDecayedAvgForEach" +"exponentialTimeDecayedAvgIf" +"exponentialTimeDecayedAvgMap" +"exponentialTimeDecayedAvgMerge" +"exponentialTimeDecayedAvgNull" +"exponentialTimeDecayedAvgOrDefault" +"exponentialTimeDecayedAvgOrNull" +"exponentialTimeDecayedAvgResample" +"exponentialTimeDecayedAvgSimpleState" +"exponentialTimeDecayedAvgState" +"exponentialTimeDecayedCount" +"exponentialTimeDecayedCountArgMax" +"exponentialTimeDecayedCountArgMin" +"exponentialTimeDecayedCountArray" +"exponentialTimeDecayedCountDistinct" +"exponentialTimeDecayedCountForEach" +"exponentialTimeDecayedCountIf" +"exponentialTimeDecayedCountMap" +"exponentialTimeDecayedCountMerge" +"exponentialTimeDecayedCountNull" +"exponentialTimeDecayedCountOrDefault" +"exponentialTimeDecayedCountOrNull" +"exponentialTimeDecayedCountResample" +"exponentialTimeDecayedCountSimpleState" +"exponentialTimeDecayedCountState" +"exponentialTimeDecayedMax" +"exponentialTimeDecayedMaxArgMax" +"exponentialTimeDecayedMaxArgMin" +"exponentialTimeDecayedMaxArray" +"exponentialTimeDecayedMaxDistinct" +"exponentialTimeDecayedMaxForEach" +"exponentialTimeDecayedMaxIf" +"exponentialTimeDecayedMaxMap" +"exponentialTimeDecayedMaxMerge" +"exponentialTimeDecayedMaxNull" +"exponentialTimeDecayedMaxOrDefault" +"exponentialTimeDecayedMaxOrNull" +"exponentialTimeDecayedMaxResample" +"exponentialTimeDecayedMaxSimpleState" +"exponentialTimeDecayedMaxState" +"exponentialTimeDecayedSum" +"exponentialTimeDecayedSumArgMax" +"exponentialTimeDecayedSumArgMin" +"exponentialTimeDecayedSumArray" +"exponentialTimeDecayedSumDistinct" +"exponentialTimeDecayedSumForEach" +"exponentialTimeDecayedSumIf" +"exponentialTimeDecayedSumMap" +"exponentialTimeDecayedSumMerge" +"exponentialTimeDecayedSumNull" +"exponentialTimeDecayedSumOrDefault" +"exponentialTimeDecayedSumOrNull" +"exponentialTimeDecayedSumResample" +"exponentialTimeDecayedSumSimpleState" +"exponentialTimeDecayedSumState" "EXPRESSION" +"EXTENDED" +"EXTERNAL DDL FROM" "extract" "EXTRACT" "extractAll" @@ -418,40 +1279,106 @@ "extractAllGroupsHorizontal" "extractAllGroupsVertical" "extractGroups" +"extractKeyValuePairs" +"extractKeyValuePairsWithEscaping" "extractTextFromHTML" "extractURLParameter" "extractURLParameterNames" "extractURLParameters" +"factorial" +"FALSE" "farmFingerprint64" "farmHash64" -"FETCHES" +"FETCH" "FETCH PART" "FETCH PARTITION" +"FIELDS" "file" +"FILE" "filesystemAvailable" +"FILESYSTEM CACHE" +"FILESYSTEM CACHES" "filesystemCapacity" -"filesystemFree" +"filesystemUnreserved" +"FILTER" "FINAL" "finalizeAggregation" "FIRST" +"firstLine" "firstSignificantSubdomain" "firstSignificantSubdomainCustom" +"firstSignificantSubdomainCustomRFC" +"firstSignificantSubdomainRFC" "first_value" +"first_valueArgMax" +"first_valueArgMin" +"first_valueArray" +"first_valueDistinct" +"first_valueForEach" +"first_valueIf" +"first_valueMap" +"first_valueMerge" +"first_valueNull" +"first_valueOrDefault" +"first_valueOrNull" +"first_valueResample" +"first_value_respect_nulls" +"first_value_respect_nullsArgMax" +"first_value_respect_nullsArgMin" +"first_value_respect_nullsArray" +"first_value_respect_nullsDistinct" +"first_value_respect_nullsForEach" +"first_value_respect_nullsIf" +"first_value_respect_nullsMap" +"first_value_respect_nullsMerge" +"first_value_respect_nullsNull" +"first_value_respect_nullsOrDefault" +"first_value_respect_nullsOrNull" +"first_value_respect_nullsResample" +"first_value_respect_nullsSimpleState" +"first_value_respect_nullsState" +"first_valueSimpleState" +"first_valueState" "FIXED" "FixedString" +"flameGraph" +"flameGraphArgMax" +"flameGraphArgMin" +"flameGraphArray" +"flameGraphDistinct" +"flameGraphForEach" +"flameGraphIf" +"flameGraphMap" +"flameGraphMerge" +"flameGraphNull" +"flameGraphOrDefault" +"flameGraphOrNull" +"flameGraphResample" +"flameGraphSimpleState" +"flameGraphState" "flatten" +"flattenTuple" "FLOAT" "Float32" "Float64" "floor" -"FLUSH" +"FOLLOWING" "FOR" "ForEach" +"FOREIGN" +"FOREIGN KEY" +"FORGET PARTITION" "format" "FORMAT" +"FORMAT_BYTES" "formatDateTime" -"formatReadableQuantity" +"formatDateTimeInJodaSyntax" +"formatQuery" +"formatQueryOrNull" +"formatQuerySingleLine" +"formatQuerySingleLineOrNull" "formatReadableDecimalSize" +"formatReadableQuantity" "formatReadableSize" "formatReadableTimeDelta" "formatRow" @@ -461,132 +1388,437 @@ "FREEZE" "FROM" "FROM_BASE64" -"toDaysSinceYearZero" +"FROM_DAYS" "fromDaysSinceYearZero" "fromDaysSinceYearZero32" +"FROM INFILE" "fromModifiedJulianDay" "fromModifiedJulianDayOrNull" +"FROM SHARD" "FROM_UNIXTIME" "fromUnixTimestamp" "fromUnixTimestamp64Micro" "fromUnixTimestamp64Milli" "fromUnixTimestamp64Nano" +"fromUnixTimestampInJodaSyntax" +"from_utc_timestamp" +"fromUTCTimestamp" "FULL" "fullHostName" +"FULLTEXT" "FUNCTION" "fuzzBits" "gccMurmurHash" "gcd" +"generateRandomStructure" +"generateULID" "generateUUIDv4" "geoDistance" "geohashDecode" "geohashEncode" "geohashesInBox" +"GEOMETRY" "geoToH3" "geoToS2" "getMacro" +"getOSKernelVersion" "__getScalar" "getServerPort" "getSetting" "getSizeOfEnumType" +"getSubcolumn" +"getTypeSerializationStreams" "GLOBAL" "globalIn" +"GLOBAL IN" "globalInIgnoreSet" "globalNotIn" +"GLOBAL NOT IN" "globalNotInIgnoreSet" "globalNotNullIn" "globalNotNullInIgnoreSet" "globalNullIn" "globalNullInIgnoreSet" "globalVariable" +"GRANT" +"GRANTEES" +"GRANT OPTION FOR" "GRANULARITY" "greatCircleAngle" "greatCircleDistance" "greater" "greaterOrEquals" "greatest" -"GROUP" "groupArray" +"groupArrayArgMax" +"groupArrayArgMin" +"groupArrayArray" +"groupArrayDistinct" +"groupArrayForEach" +"groupArrayIf" "groupArrayInsertAt" +"groupArrayInsertAtArgMax" +"groupArrayInsertAtArgMin" +"groupArrayInsertAtArray" +"groupArrayInsertAtDistinct" +"groupArrayInsertAtForEach" +"groupArrayInsertAtIf" +"groupArrayInsertAtMap" +"groupArrayInsertAtMerge" +"groupArrayInsertAtNull" +"groupArrayInsertAtOrDefault" +"groupArrayInsertAtOrNull" +"groupArrayInsertAtResample" +"groupArrayInsertAtSimpleState" +"groupArrayInsertAtState" +"groupArrayIntersect" +"groupArrayIntersectArgMax" +"groupArrayIntersectArgMin" +"groupArrayIntersectArray" +"groupArrayIntersectDistinct" +"groupArrayIntersectForEach" +"groupArrayIntersectIf" +"groupArrayIntersectMap" +"groupArrayIntersectMerge" +"groupArrayIntersectNull" +"groupArrayIntersectOrDefault" +"groupArrayIntersectOrNull" +"groupArrayIntersectResample" +"groupArrayIntersectSimpleState" +"groupArrayIntersectState" +"groupArrayLast" +"groupArrayLastArgMax" +"groupArrayLastArgMin" +"groupArrayLastArray" +"groupArrayLastDistinct" +"groupArrayLastForEach" +"groupArrayLastIf" +"groupArrayLastMap" +"groupArrayLastMerge" +"groupArrayLastNull" +"groupArrayLastOrDefault" +"groupArrayLastOrNull" +"groupArrayLastResample" +"groupArrayLastSimpleState" +"groupArrayLastState" +"groupArrayMap" +"groupArrayMerge" "groupArrayMovingAvg" +"groupArrayMovingAvgArgMax" +"groupArrayMovingAvgArgMin" +"groupArrayMovingAvgArray" +"groupArrayMovingAvgDistinct" +"groupArrayMovingAvgForEach" +"groupArrayMovingAvgIf" +"groupArrayMovingAvgMap" +"groupArrayMovingAvgMerge" +"groupArrayMovingAvgNull" +"groupArrayMovingAvgOrDefault" +"groupArrayMovingAvgOrNull" +"groupArrayMovingAvgResample" +"groupArrayMovingAvgSimpleState" +"groupArrayMovingAvgState" "groupArrayMovingSum" +"groupArrayMovingSumArgMax" +"groupArrayMovingSumArgMin" +"groupArrayMovingSumArray" +"groupArrayMovingSumDistinct" +"groupArrayMovingSumForEach" +"groupArrayMovingSumIf" +"groupArrayMovingSumMap" +"groupArrayMovingSumMerge" +"groupArrayMovingSumNull" +"groupArrayMovingSumOrDefault" +"groupArrayMovingSumOrNull" +"groupArrayMovingSumResample" +"groupArrayMovingSumSimpleState" +"groupArrayMovingSumState" +"groupArrayNull" +"groupArrayOrDefault" +"groupArrayOrNull" +"groupArrayResample" "groupArraySample" +"groupArraySampleArgMax" +"groupArraySampleArgMin" +"groupArraySampleArray" +"groupArraySampleDistinct" +"groupArraySampleForEach" +"groupArraySampleIf" +"groupArraySampleMap" +"groupArraySampleMerge" +"groupArraySampleNull" +"groupArraySampleOrDefault" +"groupArraySampleOrNull" +"groupArraySampleResample" +"groupArraySampleSimpleState" +"groupArraySampleState" +"groupArraySimpleState" +"groupArraySorted" +"groupArraySortedArgMax" +"groupArraySortedArgMin" +"groupArraySortedArray" +"groupArraySortedDistinct" +"groupArraySortedForEach" +"groupArraySortedIf" +"groupArraySortedMap" +"groupArraySortedMerge" +"groupArraySortedNull" +"groupArraySortedOrDefault" +"groupArraySortedOrNull" +"groupArraySortedResample" +"groupArraySortedSimpleState" +"groupArraySortedState" +"groupArrayState" "groupBitAnd" +"groupBitAndArgMax" +"groupBitAndArgMin" +"groupBitAndArray" +"groupBitAndDistinct" +"groupBitAndForEach" +"groupBitAndIf" +"groupBitAndMap" +"groupBitAndMerge" +"groupBitAndNull" +"groupBitAndOrDefault" +"groupBitAndOrNull" +"groupBitAndResample" +"groupBitAndSimpleState" +"groupBitAndState" "groupBitmap" "groupBitmapAnd" +"groupBitmapAndArgMax" +"groupBitmapAndArgMin" +"groupBitmapAndArray" +"groupBitmapAndDistinct" +"groupBitmapAndForEach" +"groupBitmapAndIf" +"groupBitmapAndMap" +"groupBitmapAndMerge" +"groupBitmapAndNull" +"groupBitmapAndOrDefault" +"groupBitmapAndOrNull" +"groupBitmapAndResample" +"groupBitmapAndSimpleState" +"groupBitmapAndState" +"groupBitmapArgMax" +"groupBitmapArgMin" +"groupBitmapArray" +"groupBitmapDistinct" +"groupBitmapForEach" +"groupBitmapIf" +"groupBitmapMap" +"groupBitmapMerge" +"groupBitmapNull" "groupBitmapOr" +"groupBitmapOrArgMax" +"groupBitmapOrArgMin" +"groupBitmapOrArray" +"groupBitmapOrDefault" +"groupBitmapOrDistinct" +"groupBitmapOrForEach" +"groupBitmapOrIf" +"groupBitmapOrMap" +"groupBitmapOrMerge" +"groupBitmapOrNull" +"groupBitmapOrOrDefault" +"groupBitmapOrOrNull" +"groupBitmapOrResample" +"groupBitmapOrSimpleState" +"groupBitmapOrState" +"groupBitmapResample" +"groupBitmapSimpleState" +"groupBitmapState" "groupBitmapXor" +"groupBitmapXorArgMax" +"groupBitmapXorArgMin" +"groupBitmapXorArray" +"groupBitmapXorDistinct" +"groupBitmapXorForEach" +"groupBitmapXorIf" +"groupBitmapXorMap" +"groupBitmapXorMerge" +"groupBitmapXorNull" +"groupBitmapXorOrDefault" +"groupBitmapXorOrNull" +"groupBitmapXorResample" +"groupBitmapXorSimpleState" +"groupBitmapXorState" "groupBitOr" +"groupBitOrArgMax" +"groupBitOrArgMin" +"groupBitOrArray" +"groupBitOrDistinct" +"groupBitOrForEach" +"groupBitOrIf" +"groupBitOrMap" +"groupBitOrMerge" +"groupBitOrNull" +"groupBitOrOrDefault" +"groupBitOrOrNull" +"groupBitOrResample" +"groupBitOrSimpleState" +"groupBitOrState" "groupBitXor" +"groupBitXorArgMax" +"groupBitXorArgMin" +"groupBitXorArray" +"groupBitXorDistinct" +"groupBitXorForEach" +"groupBitXorIf" +"groupBitXorMap" +"groupBitXorMerge" +"groupBitXorNull" +"groupBitXorOrDefault" +"groupBitXorOrNull" +"groupBitXorResample" +"groupBitXorSimpleState" +"groupBitXorState" "GROUP BY" +"GROUPING SETS" +"GROUPS" "groupUniqArray" +"groupUniqArrayArgMax" +"groupUniqArrayArgMin" +"groupUniqArrayArray" +"groupUniqArrayDistinct" +"groupUniqArrayForEach" +"groupUniqArrayIf" +"groupUniqArrayMap" +"groupUniqArrayMerge" +"groupUniqArrayNull" +"groupUniqArrayOrDefault" +"groupUniqArrayOrNull" +"groupUniqArrayResample" +"groupUniqArraySimpleState" +"groupUniqArrayState" +"H" +"h3CellAreaM2" +"h3CellAreaRads2" +"h3Distance" "h3EdgeAngle" +"h3EdgeLengthKm" "h3EdgeLengthM" +"h3ExactEdgeLengthKm" +"h3ExactEdgeLengthM" +"h3ExactEdgeLengthRads" "h3GetBaseCell" +"h3GetDestinationIndexFromUnidirectionalEdge" "h3GetFaces" +"h3GetIndexesFromUnidirectionalEdge" +"h3GetOriginIndexFromUnidirectionalEdge" +"h3GetPentagonIndexes" +"h3GetRes0Indexes" "h3GetResolution" +"h3GetUnidirectionalEdge" +"h3GetUnidirectionalEdgeBoundary" +"h3GetUnidirectionalEdgesFromHexagon" +"h3HexAreaKm2" "h3HexAreaM2" +"h3HexRing" "h3IndexesAreNeighbors" "h3IsPentagon" "h3IsResClassIII" "h3IsValid" "h3kRing" +"h3Line" +"h3NumHexagons" +"h3PointDistKm" +"h3PointDistM" +"h3PointDistRads" +"h3ToCenterChild" "h3ToChildren" "h3ToGeo" "h3ToGeoBoundary" "h3ToParent" "h3ToString" +"h3UnidirectionalEdgeIsValid" "halfMD5" "has" "hasAll" "hasAny" "hasColumnInTable" +"HASH" +"hasSubsequence" +"hasSubsequenceCaseInsensitive" +"hasSubsequenceCaseInsensitiveUTF8" +"hasSubsequenceUTF8" "hasSubstr" "hasThreadFuzzer" "hasToken" "hasTokenCaseInsensitive" +"hasTokenCaseInsensitiveOrNull" +"hasTokenOrNull" "HAVING" +"HDFS" "hex" "HH" "HIERARCHICAL" "histogram" +"histogramArgMax" +"histogramArgMin" +"histogramArray" +"histogramDistinct" +"histogramForEach" +"histogramIf" +"histogramMap" +"histogramMerge" +"histogramNull" +"histogramOrDefault" +"histogramOrNull" +"histogramResample" +"histogramSimpleState" +"histogramState" "hiveHash" +"hop" +"hopEnd" +"hopStart" +"HOST" "hostname" "hostName" "HOUR" +"HOURS" +"HTTP" "hypot" "ID" +"IDENTIFIED" "identity" +"idnaDecode" +"idnaEncode" "if" -"IF" +"IF EMPTY" "IF EXISTS" "IF NOT EXISTS" "ifNotFinite" "ifNull" "ignore" +"IGNORE NULLS" "ilike" "ILIKE" "in" "IN" "INDEX" +"INDEXES" "indexHint" "indexOf" +"INDICES" "INET4" "INET6" "INET6_ATON" "INET6_NTOA" "INET_ATON" "INET_NTOA" -"INF" +"INHERIT" "inIgnoreSet" +"initcap" +"initcapUTF8" "initializeAggregation" "initial_query_id" "initialQueryID" "INJECTIVE" "INNER" "IN PARTITION" -"INSERT" "INSERT INTO" +"instr" "INT" "INT1" "Int128" @@ -602,12 +1834,31 @@ "INTEGER" "INTEGER SIGNED" "INTEGER UNSIGNED" +"INTERPOLATE" +"INTERSECT" "INTERVAL" "IntervalDay" "IntervalHour" "intervalLengthSum" +"intervalLengthSumArgMax" +"intervalLengthSumArgMin" +"intervalLengthSumArray" +"intervalLengthSumDistinct" +"intervalLengthSumForEach" +"intervalLengthSumIf" +"intervalLengthSumMap" +"intervalLengthSumMerge" +"intervalLengthSumNull" +"intervalLengthSumOrDefault" +"intervalLengthSumOrNull" +"intervalLengthSumResample" +"intervalLengthSumSimpleState" +"intervalLengthSumState" +"IntervalMicrosecond" +"IntervalMillisecond" "IntervalMinute" "IntervalMonth" +"IntervalNanosecond" "IntervalQuarter" "IntervalSecond" "IntervalWeek" @@ -616,21 +1867,26 @@ "intExp2" "intHash32" "intHash64" -"INTO" "INTO OUTFILE" "INT SIGNED" "INT UNSIGNED" +"INVISIBLE" +"INVOKER" +"IP" "IPv4" "IPv4CIDRToRange" "IPv4NumToString" "IPv4NumToStringClassC" "IPv4StringToNum" +"IPv4StringToNumOrDefault" +"IPv4StringToNumOrNull" "IPv4ToIPv6" "IPv6" "IPv6CIDRToRange" "IPv6NumToString" "IPv6StringToNum" -"IS" +"IPv6StringToNumOrDefault" +"IPv6StringToNumOrNull" "isConstant" "isDecimalOverflow" "isFinite" @@ -639,64 +1895,220 @@ "isIPv4String" "isIPv6String" "isNaN" +"isNotDistinctFrom" +"IS NOT DISTINCT FROM" "isNotNull" +"IS NOT NULL" "isNull" +"IS NULL" +"isNullable" "IS_OBJECT_ID" "isValidJSON" "isValidUTF8" "isZeroOrNull" +"jaroSimilarity" +"jaroWinklerSimilarity" "javaHash" "javaHashUTF16LE" "JOIN" "joinGet" "joinGetOrNull" +"JSON" +"JSONArrayLength" +"JSON_ARRAY_LENGTH" "JSON_EXISTS" "JSONExtract" "JSONExtractArrayRaw" "JSONExtractBool" "JSONExtractFloat" "JSONExtractInt" +"JSONExtractKeys" "JSONExtractKeysAndValues" "JSONExtractKeysAndValuesRaw" -"JSONExtractKeys" "JSONExtractRaw" "JSONExtractString" "JSONExtractUInt" "JSONHas" "JSONKey" "JSONLength" +"jsonMergePatch" "JSON_QUERY" "JSONType" "JSON_VALUE" "jumpConsistentHash" +"kafkaMurmurHash" +"KERBEROS" "KEY" -# Key words (based on keywords from antlr parser) +"KEY BY" +"KEYED BY" +"KEYS" "KILL" +"KIND" +"kolmogorovSmirnovTest" +"kolmogorovSmirnovTestArgMax" +"kolmogorovSmirnovTestArgMin" +"kolmogorovSmirnovTestArray" +"kolmogorovSmirnovTestDistinct" +"kolmogorovSmirnovTestForEach" +"kolmogorovSmirnovTestIf" +"kolmogorovSmirnovTestMap" +"kolmogorovSmirnovTestMerge" +"kolmogorovSmirnovTestNull" +"kolmogorovSmirnovTestOrDefault" +"kolmogorovSmirnovTestOrNull" +"kolmogorovSmirnovTestResample" +"kolmogorovSmirnovTestSimpleState" +"kolmogorovSmirnovTestState" +"kostikConsistentHash" +"kql_array_sort_asc" +"kql_array_sort_desc" "kurtPop" +"kurtPopArgMax" +"kurtPopArgMin" +"kurtPopArray" +"kurtPopDistinct" +"kurtPopForEach" +"kurtPopIf" +"kurtPopMap" +"kurtPopMerge" +"kurtPopNull" +"kurtPopOrDefault" +"kurtPopOrNull" +"kurtPopResample" +"kurtPopSimpleState" +"kurtPopState" "kurtSamp" +"kurtSampArgMax" +"kurtSampArgMin" +"kurtSampArray" +"kurtSampDistinct" +"kurtSampForEach" +"kurtSampIf" +"kurtSampMap" +"kurtSampMerge" +"kurtSampNull" +"kurtSampOrDefault" +"kurtSampOrNull" +"kurtSampResample" +"kurtSampSimpleState" +"kurtSampState" +"L1Distance" +"L1Norm" +"L1Normalize" +"L2Distance" +"L2Norm" +"L2Normalize" +"L2SquaredDistance" +"L2SquaredNorm" "lagInFrame" +"lagInFrameArgMax" +"lagInFrameArgMin" +"lagInFrameArray" +"lagInFrameDistinct" +"lagInFrameForEach" +"lagInFrameIf" +"lagInFrameMap" +"lagInFrameMerge" +"lagInFrameNull" +"lagInFrameOrDefault" +"lagInFrameOrNull" +"lagInFrameResample" +"lagInFrameSimpleState" +"lagInFrameState" +"LARGE OBJECT" +"largestTriangleThreeBuckets" +"largestTriangleThreeBucketsArgMax" +"largestTriangleThreeBucketsArgMin" +"largestTriangleThreeBucketsArray" +"largestTriangleThreeBucketsDistinct" +"largestTriangleThreeBucketsForEach" +"largestTriangleThreeBucketsIf" +"largestTriangleThreeBucketsMap" +"largestTriangleThreeBucketsMerge" +"largestTriangleThreeBucketsNull" +"largestTriangleThreeBucketsOrDefault" +"largestTriangleThreeBucketsOrNull" +"largestTriangleThreeBucketsResample" +"largestTriangleThreeBucketsSimpleState" +"largestTriangleThreeBucketsState" "LAST" +"LAST_DAY" "last_value" +"last_valueArgMax" +"last_valueArgMin" +"last_valueArray" +"last_valueDistinct" +"last_valueForEach" +"last_valueIf" +"last_valueMap" +"last_valueMerge" +"last_valueNull" +"last_valueOrDefault" +"last_valueOrNull" +"last_valueResample" +"last_value_respect_nulls" +"last_value_respect_nullsArgMax" +"last_value_respect_nullsArgMin" +"last_value_respect_nullsArray" +"last_value_respect_nullsDistinct" +"last_value_respect_nullsForEach" +"last_value_respect_nullsIf" +"last_value_respect_nullsMap" +"last_value_respect_nullsMerge" +"last_value_respect_nullsNull" +"last_value_respect_nullsOrDefault" +"last_value_respect_nullsOrNull" +"last_value_respect_nullsResample" +"last_value_respect_nullsSimpleState" +"last_value_respect_nullsState" +"last_valueSimpleState" +"last_valueState" "LAYOUT" "lcase" "lcm" +"LDAP" "leadInFrame" +"leadInFrameArgMax" +"leadInFrameArgMin" +"leadInFrameArray" +"leadInFrameDistinct" +"leadInFrameForEach" +"leadInFrameIf" +"leadInFrameMap" +"leadInFrameMerge" +"leadInFrameNull" +"leadInFrameOrDefault" +"leadInFrameOrNull" +"leadInFrameResample" +"leadInFrameSimpleState" +"leadInFrameState" "LEADING" "least" +"left" "LEFT" "LEFT ARRAY JOIN" "leftPad" "leftPadUTF8" +"leftUTF8" "lemmatize" "length" "lengthUTF8" "less" "lessOrEquals" +"LESS THAN" +"LEVEL" +"levenshteinDistance" "lgamma" "LIFETIME" +"LIGHTWEIGHT" "like" "LIKE" "LIMIT" +"LINEAR" +"LinfDistance" +"LinfNorm" +"LinfNormalize" +"LIST" "LIVE" "ln" "LOCAL" @@ -705,7 +2117,6 @@ "log10" "log1p" "log2" -"LOGS" "logTrace" "LONGBLOB" "LONGTEXT" @@ -715,76 +2126,467 @@ "lower" "lowerUTF8" "lpad" +"LpDistance" +"LpNorm" +"LpNormalize" +"ltrim" "LTRIM" +"lttb" +"lttbArgMax" +"lttbArgMin" +"lttbArray" +"lttbDistinct" +"lttbForEach" +"lttbIf" +"lttbMap" +"lttbMerge" +"lttbNull" +"lttbOrDefault" +"lttbOrNull" +"lttbResample" +"lttbSimpleState" +"lttbState" "M" "MACNumToString" "MACStringToNum" "MACStringToOUI" +"makeDate" +"makeDate32" +"makeDateTime" +"makeDateTime64" "mannWhitneyUTest" +"mannWhitneyUTestArgMax" +"mannWhitneyUTestArgMin" +"mannWhitneyUTestArray" +"mannWhitneyUTestDistinct" +"mannWhitneyUTestForEach" +"mannWhitneyUTestIf" +"mannWhitneyUTestMap" +"mannWhitneyUTestMerge" +"mannWhitneyUTestNull" +"mannWhitneyUTestOrDefault" +"mannWhitneyUTestOrNull" +"mannWhitneyUTestResample" +"mannWhitneyUTestSimpleState" +"mannWhitneyUTestState" "map" "Map" "mapAdd" +"mapAll" +"mapApply" +"mapConcat" "mapContains" +"mapContainsKeyLike" +"mapExists" +"mapExtractKeyLike" +"mapFilter" +"mapFromArrays" +"MAP_FROM_ARRAYS" +"mapFromString" "mapKeys" +"mapPartialReverseSort" +"mapPartialSort" "mapPopulateSeries" +"mapReverseSort" +"mapSort" "mapSubtract" +"mapUpdate" "mapValues" "match" +"MATCH" "materialize" "MATERIALIZE" +"MATERIALIZE COLUMN" "MATERIALIZED" "MATERIALIZE INDEX" +"MATERIALIZE PROJECTION" +"MATERIALIZE STATISTIC" "MATERIALIZE TTL" "max" "MAX" +"max2" +"maxArgMax" +"maxArgMin" +"maxArray" +"maxDistinct" +"maxForEach" +"maxIf" "maxIntersections" +"maxIntersectionsArgMax" +"maxIntersectionsArgMin" +"maxIntersectionsArray" +"maxIntersectionsDistinct" +"maxIntersectionsForEach" +"maxIntersectionsIf" +"maxIntersectionsMap" +"maxIntersectionsMerge" +"maxIntersectionsNull" +"maxIntersectionsOrDefault" +"maxIntersectionsOrNull" "maxIntersectionsPosition" +"maxIntersectionsPositionArgMax" +"maxIntersectionsPositionArgMin" +"maxIntersectionsPositionArray" +"maxIntersectionsPositionDistinct" +"maxIntersectionsPositionForEach" +"maxIntersectionsPositionIf" +"maxIntersectionsPositionMap" +"maxIntersectionsPositionMerge" +"maxIntersectionsPositionNull" +"maxIntersectionsPositionOrDefault" +"maxIntersectionsPositionOrNull" +"maxIntersectionsPositionResample" +"maxIntersectionsPositionSimpleState" +"maxIntersectionsPositionState" +"maxIntersectionsResample" +"maxIntersectionsSimpleState" +"maxIntersectionsState" "maxMap" +"maxMappedArrays" +"maxMappedArraysArgMax" +"maxMappedArraysArgMin" +"maxMappedArraysArray" +"maxMappedArraysDistinct" +"maxMappedArraysForEach" +"maxMappedArraysIf" +"maxMappedArraysMap" +"maxMappedArraysMerge" +"maxMappedArraysNull" +"maxMappedArraysOrDefault" +"maxMappedArraysOrNull" +"maxMappedArraysResample" +"maxMappedArraysSimpleState" +"maxMappedArraysState" +"maxMerge" +"maxNull" +"maxOrDefault" +"maxOrNull" +"maxResample" +"maxSimpleState" +"maxState" +"MCS" "MD4" "MD5" +"meanZTest" +"meanZTestArgMax" +"meanZTestArgMin" +"meanZTestArray" +"meanZTestDistinct" +"meanZTestForEach" +"meanZTestIf" +"meanZTestMap" +"meanZTestMerge" +"meanZTestNull" +"meanZTestOrDefault" +"meanZTestOrNull" +"meanZTestResample" +"meanZTestSimpleState" +"meanZTestState" "median" +"medianArgMax" +"medianArgMin" +"medianArray" "medianBFloat16" +"medianBFloat16ArgMax" +"medianBFloat16ArgMin" +"medianBFloat16Array" +"medianBFloat16Distinct" +"medianBFloat16ForEach" +"medianBFloat16If" +"medianBFloat16Map" +"medianBFloat16Merge" +"medianBFloat16Null" +"medianBFloat16OrDefault" +"medianBFloat16OrNull" +"medianBFloat16Resample" +"medianBFloat16SimpleState" +"medianBFloat16State" "medianBFloat16Weighted" +"medianBFloat16WeightedArgMax" +"medianBFloat16WeightedArgMin" +"medianBFloat16WeightedArray" +"medianBFloat16WeightedDistinct" +"medianBFloat16WeightedForEach" +"medianBFloat16WeightedIf" +"medianBFloat16WeightedMap" +"medianBFloat16WeightedMerge" +"medianBFloat16WeightedNull" +"medianBFloat16WeightedOrDefault" +"medianBFloat16WeightedOrNull" +"medianBFloat16WeightedResample" +"medianBFloat16WeightedSimpleState" +"medianBFloat16WeightedState" +"medianDD" +"medianDDArgMax" +"medianDDArgMin" +"medianDDArray" +"medianDDDistinct" +"medianDDForEach" +"medianDDIf" +"medianDDMap" +"medianDDMerge" +"medianDDNull" +"medianDDOrDefault" +"medianDDOrNull" +"medianDDResample" +"medianDDSimpleState" +"medianDDState" "medianDeterministic" +"medianDeterministicArgMax" +"medianDeterministicArgMin" +"medianDeterministicArray" +"medianDeterministicDistinct" +"medianDeterministicForEach" +"medianDeterministicIf" +"medianDeterministicMap" +"medianDeterministicMerge" +"medianDeterministicNull" +"medianDeterministicOrDefault" +"medianDeterministicOrNull" +"medianDeterministicResample" +"medianDeterministicSimpleState" +"medianDeterministicState" +"medianDistinct" "medianExact" +"medianExactArgMax" +"medianExactArgMin" +"medianExactArray" +"medianExactDistinct" +"medianExactForEach" "medianExactHigh" +"medianExactHighArgMax" +"medianExactHighArgMin" +"medianExactHighArray" +"medianExactHighDistinct" +"medianExactHighForEach" +"medianExactHighIf" +"medianExactHighMap" +"medianExactHighMerge" +"medianExactHighNull" +"medianExactHighOrDefault" +"medianExactHighOrNull" +"medianExactHighResample" +"medianExactHighSimpleState" +"medianExactHighState" +"medianExactIf" "medianExactLow" +"medianExactLowArgMax" +"medianExactLowArgMin" +"medianExactLowArray" +"medianExactLowDistinct" +"medianExactLowForEach" +"medianExactLowIf" +"medianExactLowMap" +"medianExactLowMerge" +"medianExactLowNull" +"medianExactLowOrDefault" +"medianExactLowOrNull" +"medianExactLowResample" +"medianExactLowSimpleState" +"medianExactLowState" +"medianExactMap" +"medianExactMerge" +"medianExactNull" +"medianExactOrDefault" +"medianExactOrNull" +"medianExactResample" +"medianExactSimpleState" +"medianExactState" "medianExactWeighted" +"medianExactWeightedArgMax" +"medianExactWeightedArgMin" +"medianExactWeightedArray" +"medianExactWeightedDistinct" +"medianExactWeightedForEach" +"medianExactWeightedIf" +"medianExactWeightedMap" +"medianExactWeightedMerge" +"medianExactWeightedNull" +"medianExactWeightedOrDefault" +"medianExactWeightedOrNull" +"medianExactWeightedResample" +"medianExactWeightedSimpleState" +"medianExactWeightedState" +"medianForEach" +"medianGK" +"medianGKArgMax" +"medianGKArgMin" +"medianGKArray" +"medianGKDistinct" +"medianGKForEach" +"medianGKIf" +"medianGKMap" +"medianGKMerge" +"medianGKNull" +"medianGKOrDefault" +"medianGKOrNull" +"medianGKResample" +"medianGKSimpleState" +"medianGKState" +"medianIf" +"medianInterpolatedWeighted" +"medianInterpolatedWeightedArgMax" +"medianInterpolatedWeightedArgMin" +"medianInterpolatedWeightedArray" +"medianInterpolatedWeightedDistinct" +"medianInterpolatedWeightedForEach" +"medianInterpolatedWeightedIf" +"medianInterpolatedWeightedMap" +"medianInterpolatedWeightedMerge" +"medianInterpolatedWeightedNull" +"medianInterpolatedWeightedOrDefault" +"medianInterpolatedWeightedOrNull" +"medianInterpolatedWeightedResample" +"medianInterpolatedWeightedSimpleState" +"medianInterpolatedWeightedState" +"medianMap" +"medianMerge" +"medianNull" +"medianOrDefault" +"medianOrNull" +"medianResample" +"medianSimpleState" +"medianState" "medianTDigest" +"medianTDigestArgMax" +"medianTDigestArgMin" +"medianTDigestArray" +"medianTDigestDistinct" +"medianTDigestForEach" +"medianTDigestIf" +"medianTDigestMap" +"medianTDigestMerge" +"medianTDigestNull" +"medianTDigestOrDefault" +"medianTDigestOrNull" +"medianTDigestResample" +"medianTDigestSimpleState" +"medianTDigestState" "medianTDigestWeighted" +"medianTDigestWeightedArgMax" +"medianTDigestWeightedArgMin" +"medianTDigestWeightedArray" +"medianTDigestWeightedDistinct" +"medianTDigestWeightedForEach" +"medianTDigestWeightedIf" +"medianTDigestWeightedMap" +"medianTDigestWeightedMerge" +"medianTDigestWeightedNull" +"medianTDigestWeightedOrDefault" +"medianTDigestWeightedOrNull" +"medianTDigestWeightedResample" +"medianTDigestWeightedSimpleState" +"medianTDigestWeightedState" "medianTiming" +"medianTimingArgMax" +"medianTimingArgMin" +"medianTimingArray" +"medianTimingDistinct" +"medianTimingForEach" +"medianTimingIf" +"medianTimingMap" +"medianTimingMerge" +"medianTimingNull" +"medianTimingOrDefault" +"medianTimingOrNull" +"medianTimingResample" +"medianTimingSimpleState" +"medianTimingState" "medianTimingWeighted" +"medianTimingWeightedArgMax" +"medianTimingWeightedArgMin" +"medianTimingWeightedArray" +"medianTimingWeightedDistinct" +"medianTimingWeightedForEach" +"medianTimingWeightedIf" +"medianTimingWeightedMap" +"medianTimingWeightedMerge" +"medianTimingWeightedNull" +"medianTimingWeightedOrDefault" +"medianTimingWeightedOrNull" +"medianTimingWeightedResample" +"medianTimingWeightedSimpleState" +"medianTimingWeightedState" "MEDIUMBLOB" "MEDIUMINT" "MEDIUMINT SIGNED" "MEDIUMINT UNSIGNED" "MEDIUMTEXT" +"MEMORY" "Merge" "MERGES" "metroHash64" "MI" +"MICROSECOND" +"MICROSECONDS" "mid" +"MILLISECOND" +"MILLISECONDS" "min" "MIN" +"min2" +"minArgMax" +"minArgMin" +"minArray" +"minDistinct" +"minForEach" +"minIf" "minMap" +"minMappedArrays" +"minMappedArraysArgMax" +"minMappedArraysArgMin" +"minMappedArraysArray" +"minMappedArraysDistinct" +"minMappedArraysForEach" +"minMappedArraysIf" +"minMappedArraysMap" +"minMappedArraysMerge" +"minMappedArraysNull" +"minMappedArraysOrDefault" +"minMappedArraysOrNull" +"minMappedArraysResample" +"minMappedArraysSimpleState" +"minMappedArraysState" +"minMerge" +"minNull" +"minOrDefault" +"minOrNull" +"minResample" +"minSampleSizeContinous" +"minSampleSizeContinuous" +"minSampleSizeConversion" +"minSimpleState" +"minState" "minus" "MINUTE" +"MINUTES" +"mismatches" "MM" "mod" +"MOD" "MODIFY" "MODIFY COLUMN" +"MODIFY COMMENT" "MODIFY ORDER BY" "MODIFY QUERY" +"MODIFY REFRESH" +"MODIFY SAMPLE BY" "MODIFY SETTING" +"MODIFY SQL SECURITY" "MODIFY TTL" "modulo" "moduloLegacy" "moduloOrZero" "MONTH" +"monthName" +"MONTHS" +"mortonDecode" +"mortonEncode" "MOVE" "MOVE PART" "MOVE PARTITION" "movingXXX" +"MS" "multiFuzzyMatchAllIndices" "multiFuzzyMatchAny" "multiFuzzyMatchAnyIndex" @@ -793,6 +2595,7 @@ "multiMatchAny" "multiMatchAnyIndex" "multiply" +"multiplyDecimal" "MultiPolygon" "multiSearchAllPositions" "multiSearchAllPositionsCaseInsensitive" @@ -818,7 +2621,9 @@ "MUTATION" "N" "NAME" -"NAN_SQL" +"NAMED COLLECTION" +"NANOSECOND" +"NANOSECONDS" "NATIONAL CHAR" "NATIONAL CHARACTER" "NATIONAL CHARACTER LARGE OBJECT" @@ -829,8 +2634,10 @@ "NCHAR VARYING" "negate" "neighbor" +"nested" "Nested" "netloc" +"NEXT" "ngramDistance" "ngramDistanceCaseInsensitive" "ngramDistanceCaseInsensitiveUTF8" @@ -843,6 +2650,7 @@ "ngramMinHashCaseInsensitive" "ngramMinHashCaseInsensitiveUTF8" "ngramMinHashUTF8" +"ngrams" "ngramSearch" "ngramSearchCaseInsensitive" "ngramSearchCaseInsensitiveUTF8" @@ -851,68 +2659,207 @@ "ngramSimHashCaseInsensitive" "ngramSimHashCaseInsensitiveUTF8" "ngramSimHashUTF8" -"NO" +"NO ACTION" "NO DELAY" +"NO LIMITS" "NONE" +"nonNegativeDerivative" +"nonNegativeDerivativeArgMax" +"nonNegativeDerivativeArgMin" +"nonNegativeDerivativeArray" +"nonNegativeDerivativeDistinct" +"nonNegativeDerivativeForEach" +"nonNegativeDerivativeIf" +"nonNegativeDerivativeMap" +"nonNegativeDerivativeMerge" +"nonNegativeDerivativeNull" +"nonNegativeDerivativeOrDefault" +"nonNegativeDerivativeOrNull" +"nonNegativeDerivativeResample" +"nonNegativeDerivativeSimpleState" +"nonNegativeDerivativeState" +"NO_PASSWORD" "normalizedQueryHash" "normalizedQueryHashKeepNames" +"normalizeL1" +"normalizeL2" +"normalizeLinf" +"normalizeLp" "normalizeQuery" "normalizeQueryKeepNames" +"normalizeUTF8NFC" +"normalizeUTF8NFD" +"normalizeUTF8NFKC" +"normalizeUTF8NFKD" +"normL1" +"normL2" +"normL2Squared" +"normLinf" +"normLp" "not" "NOT" +"NOT BETWEEN" "notEmpty" "notEquals" "nothing" "Nothing" +"nothingArgMax" +"nothingArgMin" +"nothingArray" +"nothingDistinct" +"nothingForEach" +"nothingIf" +"nothingMap" +"nothingMerge" +"nothingNull" +"nothingNullArgMax" +"nothingNullArgMin" +"nothingNullArray" +"nothingNullDistinct" +"nothingNullForEach" +"nothingNullIf" +"nothingNullMap" +"nothingNullMerge" +"nothingNullNull" +"nothingNullOrDefault" +"nothingNullOrNull" +"nothingNullResample" +"nothingNullSimpleState" +"nothingNullState" +"nothingOrDefault" +"nothingOrNull" +"nothingResample" +"nothingSimpleState" +"nothingState" +"nothingUInt64" +"nothingUInt64ArgMax" +"nothingUInt64ArgMin" +"nothingUInt64Array" +"nothingUInt64Distinct" +"nothingUInt64ForEach" +"nothingUInt64If" +"nothingUInt64Map" +"nothingUInt64Merge" +"nothingUInt64Null" +"nothingUInt64OrDefault" +"nothingUInt64OrNull" +"nothingUInt64Resample" +"nothingUInt64SimpleState" +"nothingUInt64State" +"NOT IDENTIFIED" "notILike" +"NOT ILIKE" "notIn" +"NOT IN" "notInIgnoreSet" +"NOT KEYED" "notLike" +"NOT LIKE" "notNullIn" "notNullInIgnoreSet" +"NOT OVERRIDABLE" "now" "now64" +"nowInBlock" +"NS" +"nth_value" +"nth_valueArgMax" +"nth_valueArgMin" +"nth_valueArray" +"nth_valueDistinct" +"nth_valueForEach" +"nth_valueIf" +"nth_valueMap" +"nth_valueMerge" +"nth_valueNull" +"nth_valueOrDefault" +"nth_valueOrNull" +"nth_valueResample" +"nth_valueSimpleState" +"nth_valueState" +"ntile" +"ntileArgMax" +"ntileArgMin" +"ntileArray" +"ntileDistinct" +"ntileForEach" +"ntileIf" +"ntileMap" +"ntileMerge" +"ntileNull" +"ntileOrDefault" +"ntileOrNull" +"ntileResample" +"ntileSimpleState" +"ntileState" "Null" +"NULL" "Nullable" "nullIf" "nullIn" "nullInIgnoreSet" "NULLS" -"NULL_SQL" "NUMERIC" "NVARCHAR" +"Object" +"OCTET_LENGTH" "OFFSET" "ON" +"ON DELETE" "ONLY" -"OPTIMIZE" +"ON UPDATE" +"ON VOLUME" "OPTIMIZE TABLE" "or" "OR" -"ORDER" "ORDER BY" "OR REPLACE" "OUTER" -"OUTFILE" +"OVER" +"OVERRIDABLE" +"parseDateTime" "parseDateTime32BestEffort" "parseDateTime32BestEffortOrNull" "parseDateTime32BestEffortOrZero" "parseDateTime64BestEffort" "parseDateTime64BestEffortOrNull" "parseDateTime64BestEffortOrZero" +"parseDateTime64BestEffortUS" +"parseDateTime64BestEffortUSOrNull" +"parseDateTime64BestEffortUSOrZero" "parseDateTimeBestEffort" "parseDateTimeBestEffortOrNull" "parseDateTimeBestEffortOrZero" "parseDateTimeBestEffortUS" "parseDateTimeBestEffortUSOrNull" "parseDateTimeBestEffortUSOrZero" +"parseDateTimeInJodaSyntax" +"parseDateTimeInJodaSyntaxOrNull" +"parseDateTimeInJodaSyntaxOrZero" +"parseDateTimeOrNull" +"parseDateTimeOrZero" "parseTimeDelta" +"PART" +"PARTIAL" "PARTITION" "PARTITION BY" "partitionId" +"PARTITIONS" +"PART_MOVE_TO_SHARD" +"PASTE" "path" "pathFull" +"PERIODIC REFRESH" +"PERMANENTLY" +"PERMISSIVE" +"PERSISTENT" +"PERSISTENT SEQUENTIAL" "pi" +"PIPELINE" +"PLAINTEXT_PASSWORD" +"PLAN" "plus" +"pmod" "Point" "pointInEllipses" "pointInPolygon" @@ -935,71 +2882,620 @@ "polygonsWithinSpherical" "POPULATE" "port" +"portRFC" "position" "positionCaseInsensitive" "positionCaseInsensitiveUTF8" "positionUTF8" +"positive_modulo" +"positiveModulo" "pow" "power" +"PRECEDING" +"PRECISION" "PREWHERE" "PRIMARY" "PRIMARY KEY" +"PROFILE" "PROJECTION" +"proportionsZTest" +"Protobuf" "protocol" +"PULL" +"punycodeDecode" +"punycodeEncode" "Q" "QQ" "quantile" +"quantileArgMax" +"quantileArgMin" +"quantileArray" "quantileBFloat16" +"quantileBFloat16ArgMax" +"quantileBFloat16ArgMin" +"quantileBFloat16Array" +"quantileBFloat16Distinct" +"quantileBFloat16ForEach" +"quantileBFloat16If" +"quantileBFloat16Map" +"quantileBFloat16Merge" +"quantileBFloat16Null" +"quantileBFloat16OrDefault" +"quantileBFloat16OrNull" +"quantileBFloat16Resample" +"quantileBFloat16SimpleState" +"quantileBFloat16State" "quantileBFloat16Weighted" +"quantileBFloat16WeightedArgMax" +"quantileBFloat16WeightedArgMin" +"quantileBFloat16WeightedArray" +"quantileBFloat16WeightedDistinct" +"quantileBFloat16WeightedForEach" +"quantileBFloat16WeightedIf" +"quantileBFloat16WeightedMap" +"quantileBFloat16WeightedMerge" +"quantileBFloat16WeightedNull" +"quantileBFloat16WeightedOrDefault" +"quantileBFloat16WeightedOrNull" +"quantileBFloat16WeightedResample" +"quantileBFloat16WeightedSimpleState" +"quantileBFloat16WeightedState" +"quantileDD" +"quantileDDArgMax" +"quantileDDArgMin" +"quantileDDArray" +"quantileDDDistinct" +"quantileDDForEach" +"quantileDDIf" +"quantileDDMap" +"quantileDDMerge" +"quantileDDNull" +"quantileDDOrDefault" +"quantileDDOrNull" +"quantileDDResample" +"quantileDDSimpleState" +"quantileDDState" "quantileDeterministic" +"quantileDeterministicArgMax" +"quantileDeterministicArgMin" +"quantileDeterministicArray" +"quantileDeterministicDistinct" +"quantileDeterministicForEach" +"quantileDeterministicIf" +"quantileDeterministicMap" +"quantileDeterministicMerge" +"quantileDeterministicNull" +"quantileDeterministicOrDefault" +"quantileDeterministicOrNull" +"quantileDeterministicResample" +"quantileDeterministicSimpleState" +"quantileDeterministicState" +"quantileDistinct" "quantileExact" +"quantileExactArgMax" +"quantileExactArgMin" +"quantileExactArray" +"quantileExactDistinct" "quantileExactExclusive" +"quantileExactExclusiveArgMax" +"quantileExactExclusiveArgMin" +"quantileExactExclusiveArray" +"quantileExactExclusiveDistinct" +"quantileExactExclusiveForEach" +"quantileExactExclusiveIf" +"quantileExactExclusiveMap" +"quantileExactExclusiveMerge" +"quantileExactExclusiveNull" +"quantileExactExclusiveOrDefault" +"quantileExactExclusiveOrNull" +"quantileExactExclusiveResample" +"quantileExactExclusiveSimpleState" +"quantileExactExclusiveState" +"quantileExactForEach" "quantileExactHigh" +"quantileExactHighArgMax" +"quantileExactHighArgMin" +"quantileExactHighArray" +"quantileExactHighDistinct" +"quantileExactHighForEach" +"quantileExactHighIf" +"quantileExactHighMap" +"quantileExactHighMerge" +"quantileExactHighNull" +"quantileExactHighOrDefault" +"quantileExactHighOrNull" +"quantileExactHighResample" +"quantileExactHighSimpleState" +"quantileExactHighState" +"quantileExactIf" "quantileExactInclusive" +"quantileExactInclusiveArgMax" +"quantileExactInclusiveArgMin" +"quantileExactInclusiveArray" +"quantileExactInclusiveDistinct" +"quantileExactInclusiveForEach" +"quantileExactInclusiveIf" +"quantileExactInclusiveMap" +"quantileExactInclusiveMerge" +"quantileExactInclusiveNull" +"quantileExactInclusiveOrDefault" +"quantileExactInclusiveOrNull" +"quantileExactInclusiveResample" +"quantileExactInclusiveSimpleState" +"quantileExactInclusiveState" "quantileExactLow" +"quantileExactLowArgMax" +"quantileExactLowArgMin" +"quantileExactLowArray" +"quantileExactLowDistinct" +"quantileExactLowForEach" +"quantileExactLowIf" +"quantileExactLowMap" +"quantileExactLowMerge" +"quantileExactLowNull" +"quantileExactLowOrDefault" +"quantileExactLowOrNull" +"quantileExactLowResample" +"quantileExactLowSimpleState" +"quantileExactLowState" +"quantileExactMap" +"quantileExactMerge" +"quantileExactNull" +"quantileExactOrDefault" +"quantileExactOrNull" +"quantileExactResample" +"quantileExactSimpleState" +"quantileExactState" "quantileExactWeighted" +"quantileExactWeightedArgMax" +"quantileExactWeightedArgMin" +"quantileExactWeightedArray" +"quantileExactWeightedDistinct" +"quantileExactWeightedForEach" +"quantileExactWeightedIf" +"quantileExactWeightedMap" +"quantileExactWeightedMerge" +"quantileExactWeightedNull" +"quantileExactWeightedOrDefault" +"quantileExactWeightedOrNull" +"quantileExactWeightedResample" +"quantileExactWeightedSimpleState" +"quantileExactWeightedState" +"quantileForEach" +"quantileGK" +"quantileGKArgMax" +"quantileGKArgMin" +"quantileGKArray" +"quantileGKDistinct" +"quantileGKForEach" +"quantileGKIf" +"quantileGKMap" +"quantileGKMerge" +"quantileGKNull" +"quantileGKOrDefault" +"quantileGKOrNull" +"quantileGKResample" +"quantileGKSimpleState" +"quantileGKState" +"quantileIf" +"quantileInterpolatedWeighted" +"quantileInterpolatedWeightedArgMax" +"quantileInterpolatedWeightedArgMin" +"quantileInterpolatedWeightedArray" +"quantileInterpolatedWeightedDistinct" +"quantileInterpolatedWeightedForEach" +"quantileInterpolatedWeightedIf" +"quantileInterpolatedWeightedMap" +"quantileInterpolatedWeightedMerge" +"quantileInterpolatedWeightedNull" +"quantileInterpolatedWeightedOrDefault" +"quantileInterpolatedWeightedOrNull" +"quantileInterpolatedWeightedResample" +"quantileInterpolatedWeightedSimpleState" +"quantileInterpolatedWeightedState" +"quantileMap" +"quantileMerge" +"quantileNull" +"quantileOrDefault" +"quantileOrNull" +"quantileResample" "quantiles" +"quantilesArgMax" +"quantilesArgMin" +"quantilesArray" "quantilesBFloat16" +"quantilesBFloat16ArgMax" +"quantilesBFloat16ArgMin" +"quantilesBFloat16Array" +"quantilesBFloat16Distinct" +"quantilesBFloat16ForEach" +"quantilesBFloat16If" +"quantilesBFloat16Map" +"quantilesBFloat16Merge" +"quantilesBFloat16Null" +"quantilesBFloat16OrDefault" +"quantilesBFloat16OrNull" +"quantilesBFloat16Resample" +"quantilesBFloat16SimpleState" +"quantilesBFloat16State" "quantilesBFloat16Weighted" +"quantilesBFloat16WeightedArgMax" +"quantilesBFloat16WeightedArgMin" +"quantilesBFloat16WeightedArray" +"quantilesBFloat16WeightedDistinct" +"quantilesBFloat16WeightedForEach" +"quantilesBFloat16WeightedIf" +"quantilesBFloat16WeightedMap" +"quantilesBFloat16WeightedMerge" +"quantilesBFloat16WeightedNull" +"quantilesBFloat16WeightedOrDefault" +"quantilesBFloat16WeightedOrNull" +"quantilesBFloat16WeightedResample" +"quantilesBFloat16WeightedSimpleState" +"quantilesBFloat16WeightedState" +"quantilesDD" +"quantilesDDArgMax" +"quantilesDDArgMin" +"quantilesDDArray" +"quantilesDDDistinct" +"quantilesDDForEach" +"quantilesDDIf" +"quantilesDDMap" +"quantilesDDMerge" +"quantilesDDNull" +"quantilesDDOrDefault" +"quantilesDDOrNull" +"quantilesDDResample" +"quantilesDDSimpleState" +"quantilesDDState" "quantilesDeterministic" +"quantilesDeterministicArgMax" +"quantilesDeterministicArgMin" +"quantilesDeterministicArray" +"quantilesDeterministicDistinct" +"quantilesDeterministicForEach" +"quantilesDeterministicIf" +"quantilesDeterministicMap" +"quantilesDeterministicMerge" +"quantilesDeterministicNull" +"quantilesDeterministicOrDefault" +"quantilesDeterministicOrNull" +"quantilesDeterministicResample" +"quantilesDeterministicSimpleState" +"quantilesDeterministicState" +"quantilesDistinct" "quantilesExact" +"quantilesExactArgMax" +"quantilesExactArgMin" +"quantilesExactArray" +"quantilesExactDistinct" "quantilesExactExclusive" +"quantilesExactExclusiveArgMax" +"quantilesExactExclusiveArgMin" +"quantilesExactExclusiveArray" +"quantilesExactExclusiveDistinct" +"quantilesExactExclusiveForEach" +"quantilesExactExclusiveIf" +"quantilesExactExclusiveMap" +"quantilesExactExclusiveMerge" +"quantilesExactExclusiveNull" +"quantilesExactExclusiveOrDefault" +"quantilesExactExclusiveOrNull" +"quantilesExactExclusiveResample" +"quantilesExactExclusiveSimpleState" +"quantilesExactExclusiveState" +"quantilesExactForEach" "quantilesExactHigh" +"quantilesExactHighArgMax" +"quantilesExactHighArgMin" +"quantilesExactHighArray" +"quantilesExactHighDistinct" +"quantilesExactHighForEach" +"quantilesExactHighIf" +"quantilesExactHighMap" +"quantilesExactHighMerge" +"quantilesExactHighNull" +"quantilesExactHighOrDefault" +"quantilesExactHighOrNull" +"quantilesExactHighResample" +"quantilesExactHighSimpleState" +"quantilesExactHighState" +"quantilesExactIf" "quantilesExactInclusive" +"quantilesExactInclusiveArgMax" +"quantilesExactInclusiveArgMin" +"quantilesExactInclusiveArray" +"quantilesExactInclusiveDistinct" +"quantilesExactInclusiveForEach" +"quantilesExactInclusiveIf" +"quantilesExactInclusiveMap" +"quantilesExactInclusiveMerge" +"quantilesExactInclusiveNull" +"quantilesExactInclusiveOrDefault" +"quantilesExactInclusiveOrNull" +"quantilesExactInclusiveResample" +"quantilesExactInclusiveSimpleState" +"quantilesExactInclusiveState" "quantilesExactLow" +"quantilesExactLowArgMax" +"quantilesExactLowArgMin" +"quantilesExactLowArray" +"quantilesExactLowDistinct" +"quantilesExactLowForEach" +"quantilesExactLowIf" +"quantilesExactLowMap" +"quantilesExactLowMerge" +"quantilesExactLowNull" +"quantilesExactLowOrDefault" +"quantilesExactLowOrNull" +"quantilesExactLowResample" +"quantilesExactLowSimpleState" +"quantilesExactLowState" +"quantilesExactMap" +"quantilesExactMerge" +"quantilesExactNull" +"quantilesExactOrDefault" +"quantilesExactOrNull" +"quantilesExactResample" +"quantilesExactSimpleState" +"quantilesExactState" "quantilesExactWeighted" +"quantilesExactWeightedArgMax" +"quantilesExactWeightedArgMin" +"quantilesExactWeightedArray" +"quantilesExactWeightedDistinct" +"quantilesExactWeightedForEach" +"quantilesExactWeightedIf" +"quantilesExactWeightedMap" +"quantilesExactWeightedMerge" +"quantilesExactWeightedNull" +"quantilesExactWeightedOrDefault" +"quantilesExactWeightedOrNull" +"quantilesExactWeightedResample" +"quantilesExactWeightedSimpleState" +"quantilesExactWeightedState" +"quantilesForEach" +"quantilesGK" +"quantilesGKArgMax" +"quantilesGKArgMin" +"quantilesGKArray" +"quantilesGKDistinct" +"quantilesGKForEach" +"quantilesGKIf" +"quantilesGKMap" +"quantilesGKMerge" +"quantilesGKNull" +"quantilesGKOrDefault" +"quantilesGKOrNull" +"quantilesGKResample" +"quantilesGKSimpleState" +"quantilesGKState" +"quantilesIf" +"quantileSimpleState" +"quantilesInterpolatedWeighted" +"quantilesInterpolatedWeightedArgMax" +"quantilesInterpolatedWeightedArgMin" +"quantilesInterpolatedWeightedArray" +"quantilesInterpolatedWeightedDistinct" +"quantilesInterpolatedWeightedForEach" +"quantilesInterpolatedWeightedIf" +"quantilesInterpolatedWeightedMap" +"quantilesInterpolatedWeightedMerge" +"quantilesInterpolatedWeightedNull" +"quantilesInterpolatedWeightedOrDefault" +"quantilesInterpolatedWeightedOrNull" +"quantilesInterpolatedWeightedResample" +"quantilesInterpolatedWeightedSimpleState" +"quantilesInterpolatedWeightedState" +"quantilesMap" +"quantilesMerge" +"quantilesNull" +"quantilesOrDefault" +"quantilesOrNull" +"quantilesResample" +"quantilesSimpleState" +"quantilesState" +"quantileState" "quantilesTDigest" +"quantilesTDigestArgMax" +"quantilesTDigestArgMin" +"quantilesTDigestArray" +"quantilesTDigestDistinct" +"quantilesTDigestForEach" +"quantilesTDigestIf" +"quantilesTDigestMap" +"quantilesTDigestMerge" +"quantilesTDigestNull" +"quantilesTDigestOrDefault" +"quantilesTDigestOrNull" +"quantilesTDigestResample" +"quantilesTDigestSimpleState" +"quantilesTDigestState" "quantilesTDigestWeighted" +"quantilesTDigestWeightedArgMax" +"quantilesTDigestWeightedArgMin" +"quantilesTDigestWeightedArray" +"quantilesTDigestWeightedDistinct" +"quantilesTDigestWeightedForEach" +"quantilesTDigestWeightedIf" +"quantilesTDigestWeightedMap" +"quantilesTDigestWeightedMerge" +"quantilesTDigestWeightedNull" +"quantilesTDigestWeightedOrDefault" +"quantilesTDigestWeightedOrNull" +"quantilesTDigestWeightedResample" +"quantilesTDigestWeightedSimpleState" +"quantilesTDigestWeightedState" "quantilesTiming" +"quantilesTimingArgMax" +"quantilesTimingArgMin" +"quantilesTimingArray" +"quantilesTimingDistinct" +"quantilesTimingForEach" +"quantilesTimingIf" +"quantilesTimingMap" +"quantilesTimingMerge" +"quantilesTimingNull" +"quantilesTimingOrDefault" +"quantilesTimingOrNull" +"quantilesTimingResample" +"quantilesTimingSimpleState" +"quantilesTimingState" "quantilesTimingWeighted" +"quantilesTimingWeightedArgMax" +"quantilesTimingWeightedArgMin" +"quantilesTimingWeightedArray" +"quantilesTimingWeightedDistinct" +"quantilesTimingWeightedForEach" +"quantilesTimingWeightedIf" +"quantilesTimingWeightedMap" +"quantilesTimingWeightedMerge" +"quantilesTimingWeightedNull" +"quantilesTimingWeightedOrDefault" +"quantilesTimingWeightedOrNull" +"quantilesTimingWeightedResample" +"quantilesTimingWeightedSimpleState" +"quantilesTimingWeightedState" "quantileTDigest" +"quantileTDigestArgMax" +"quantileTDigestArgMin" +"quantileTDigestArray" +"quantileTDigestDistinct" +"quantileTDigestForEach" +"quantileTDigestIf" +"quantileTDigestMap" +"quantileTDigestMerge" +"quantileTDigestNull" +"quantileTDigestOrDefault" +"quantileTDigestOrNull" +"quantileTDigestResample" +"quantileTDigestSimpleState" +"quantileTDigestState" "quantileTDigestWeighted" +"quantileTDigestWeightedArgMax" +"quantileTDigestWeightedArgMin" +"quantileTDigestWeightedArray" +"quantileTDigestWeightedDistinct" +"quantileTDigestWeightedForEach" +"quantileTDigestWeightedIf" +"quantileTDigestWeightedMap" +"quantileTDigestWeightedMerge" +"quantileTDigestWeightedNull" +"quantileTDigestWeightedOrDefault" +"quantileTDigestWeightedOrNull" +"quantileTDigestWeightedResample" +"quantileTDigestWeightedSimpleState" +"quantileTDigestWeightedState" "quantileTiming" +"quantileTimingArgMax" +"quantileTimingArgMin" +"quantileTimingArray" +"quantileTimingDistinct" +"quantileTimingForEach" +"quantileTimingIf" +"quantileTimingMap" +"quantileTimingMerge" +"quantileTimingNull" +"quantileTimingOrDefault" +"quantileTimingOrNull" +"quantileTimingResample" +"quantileTimingSimpleState" +"quantileTimingState" "quantileTimingWeighted" +"quantileTimingWeightedArgMax" +"quantileTimingWeightedArgMin" +"quantileTimingWeightedArray" +"quantileTimingWeightedDistinct" +"quantileTimingWeightedForEach" +"quantileTimingWeightedIf" +"quantileTimingWeightedMap" +"quantileTimingWeightedMerge" +"quantileTimingWeightedNull" +"quantileTimingWeightedOrDefault" +"quantileTimingWeightedOrNull" +"quantileTimingWeightedResample" +"quantileTimingWeightedSimpleState" +"quantileTimingWeightedState" "QUARTER" +"QUARTERS" +"QUERY" "query_id" "queryID" "queryString" "queryStringAndFragment" +"QUERY TREE" +"QUOTA" +"radians" "rand" "rand32" "rand64" +"randBernoulli" +"randBinomial" +"randCanonical" +"randChiSquared" "randConstant" +"randExponential" +"randFisherF" +"randLogNormal" +"randNegativeBinomial" +"randNormal" "randomFixedString" +"RANDOMIZED" +"RANDOMIZE FOR" "randomPrintableASCII" "randomString" "randomStringUTF8" +"randPoisson" +"randStudentT" +"randUniform" "range" "RANGE" "rank" +"rankArgMax" +"rankArgMin" +"rankArray" "rankCorr" +"rankCorrArgMax" +"rankCorrArgMin" +"rankCorrArray" +"rankCorrDistinct" +"rankCorrForEach" +"rankCorrIf" +"rankCorrMap" +"rankCorrMerge" +"rankCorrNull" +"rankCorrOrDefault" +"rankCorrOrNull" +"rankCorrResample" +"rankCorrSimpleState" +"rankCorrState" +"rankDistinct" +"rankForEach" +"rankIf" +"rankMap" +"rankMerge" +"rankNull" +"rankOrDefault" +"rankOrNull" +"rankResample" +"rankSimpleState" +"rankState" +"READONLY" "readWKTMultiPolygon" "readWKTPoint" "readWKTPolygon" "readWKTRing" "REAL" +"REALM" +"RECOMPRESS" +"REFERENCES" "REFRESH" +"REGEXP" +"regexpExtract" +"REGEXP_EXTRACT" +"REGEXP_MATCHES" "regexpQuoteMeta" +"REGEXP_REPLACE" "regionHierarchy" "regionIn" "regionToArea" @@ -1030,11 +3526,15 @@ "reinterpretAsUInt64" "reinterpretAsUInt8" "reinterpretAsUUID" -"RELOAD" "REMOVE" +"REMOVE SAMPLE BY" +"REMOVE TTL" "RENAME" "RENAME COLUMN" +"RENAME DATABASE" +"RENAME DICTIONARY" "RENAME TABLE" +"RENAME TO" "repeat" "replace" "REPLACE" @@ -1043,18 +3543,40 @@ "REPLACE PARTITION" "replaceRegexpAll" "replaceRegexpOne" -"REPLICA" "replicate" -"REPLICATED" "Resample" +"RESET SETTING" +"RESPECT NULLS" +"RESTORE" +"RESTRICT" +"RESTRICTIVE" "RESUME" "retention" +"retentionArgMax" +"retentionArgMin" +"retentionArray" +"retentionDistinct" +"retentionForEach" +"retentionIf" +"retentionMap" +"retentionMerge" +"retentionNull" +"retentionOrDefault" +"retentionOrNull" +"retentionResample" +"retentionSimpleState" +"retentionState" "reverse" "reverseUTF8" +"revision" +"REVOKE" +"right" "RIGHT" "rightPad" "rightPadUTF8" +"rightUTF8" "Ring" +"ROLLBACK" "ROLLUP" "round" "roundAge" @@ -1062,10 +3584,27 @@ "roundDown" "roundDuration" "roundToExp2" +"ROW" "row_number" +"row_numberArgMax" +"row_numberArgMin" +"row_numberArray" +"row_numberDistinct" +"row_numberForEach" +"row_numberIf" "rowNumberInAllBlocks" "rowNumberInBlock" +"row_numberMap" +"row_numberMerge" +"row_numberNull" +"row_numberOrDefault" +"row_numberOrNull" +"row_numberResample" +"row_numberSimpleState" +"row_numberState" +"ROWS" "rpad" +"rtrim" "RTRIM" "runningAccumulate" "runningConcurrency" @@ -1081,29 +3620,103 @@ "s2RectIntersection" "s2RectUnion" "s2ToGeo" +"S3" +"SALT" "SAMPLE" "SAMPLE BY" +"scalarProduct" +"__scalarSubqueryResult" +"SCHEMA" +"SCHEME" "SECOND" +"SECONDS" "SELECT" "SEMI" -"SENDS" "sequenceCount" +"sequenceCountArgMax" +"sequenceCountArgMin" +"sequenceCountArray" +"sequenceCountDistinct" +"sequenceCountForEach" +"sequenceCountIf" +"sequenceCountMap" +"sequenceCountMerge" +"sequenceCountNull" +"sequenceCountOrDefault" +"sequenceCountOrNull" +"sequenceCountResample" +"sequenceCountSimpleState" +"sequenceCountState" "sequenceMatch" +"sequenceMatchArgMax" +"sequenceMatchArgMin" +"sequenceMatchArray" +"sequenceMatchDistinct" +"sequenceMatchForEach" +"sequenceMatchIf" +"sequenceMatchMap" +"sequenceMatchMerge" +"sequenceMatchNull" +"sequenceMatchOrDefault" +"sequenceMatchOrNull" +"sequenceMatchResample" +"sequenceMatchSimpleState" +"sequenceMatchState" "sequenceNextNode" +"sequenceNextNodeArgMax" +"sequenceNextNodeArgMin" +"sequenceNextNodeArray" +"sequenceNextNodeDistinct" +"sequenceNextNodeForEach" +"sequenceNextNodeIf" +"sequenceNextNodeMap" +"sequenceNextNodeMerge" +"sequenceNextNodeNull" +"sequenceNextNodeOrDefault" +"sequenceNextNodeOrNull" +"sequenceNextNodeResample" +"sequenceNextNodeSimpleState" +"sequenceNextNodeState" +"seriesDecomposeSTL" +"seriesOutliersDetectTukey" +"seriesPeriodDetectFFT" +"SERVER" +"serverTimezone" +"serverTimeZone" "serverUUID" "SET" +"SET DEFAULT" +"SET DEFAULT ROLE" +"SET FAKE TIME" +"SET NULL" +"SET ROLE" +"SET ROLE DEFAULT" "SETTINGS" +"SET TRANSACTION SNAPSHOT" "SHA1" "SHA224" "SHA256" +"SHA256_HASH" +"SHA256_PASSWORD" "SHA384" "SHA512" +"SHA512_256" "shardCount" "shardNum" "SHOW" +"SHOW ACCESS" +"showCertificate" +"SHOW CREATE" +"SHOW ENGINES" +"SHOW FUNCTIONS" +"SHOW GRANTS" +"SHOW PRIVILEGES" "SHOW PROCESSLIST" +"SHOW SETTING" "sigmoid" "sign" +"SIGNED" +"SIMPLE" "SimpleAggregateFunction" "simpleJSONExtractBool" "simpleJSONExtractFloat" @@ -1113,14 +3726,74 @@ "simpleJSONExtractUInt" "simpleJSONHas" "simpleLinearRegression" +"simpleLinearRegressionArgMax" +"simpleLinearRegressionArgMin" +"simpleLinearRegressionArray" +"simpleLinearRegressionDistinct" +"simpleLinearRegressionForEach" +"simpleLinearRegressionIf" +"simpleLinearRegressionMap" +"simpleLinearRegressionMerge" +"simpleLinearRegressionNull" +"simpleLinearRegressionOrDefault" +"simpleLinearRegressionOrNull" +"simpleLinearRegressionResample" +"simpleLinearRegressionSimpleState" +"simpleLinearRegressionState" "sin" "SINGLE" "singleValueOrNull" +"singleValueOrNullArgMax" +"singleValueOrNullArgMin" +"singleValueOrNullArray" +"singleValueOrNullDistinct" +"singleValueOrNullForEach" +"singleValueOrNullIf" +"singleValueOrNullMap" +"singleValueOrNullMerge" +"singleValueOrNullNull" +"singleValueOrNullOrDefault" +"singleValueOrNullOrNull" +"singleValueOrNullResample" +"singleValueOrNullSimpleState" +"singleValueOrNullState" "sinh" "sipHash128" +"sipHash128Keyed" +"sipHash128Reference" +"sipHash128ReferenceKeyed" "sipHash64" +"sipHash64Keyed" "skewPop" +"skewPopArgMax" +"skewPopArgMin" +"skewPopArray" +"skewPopDistinct" +"skewPopForEach" +"skewPopIf" +"skewPopMap" +"skewPopMerge" +"skewPopNull" +"skewPopOrDefault" +"skewPopOrNull" +"skewPopResample" +"skewPopSimpleState" +"skewPopState" "skewSamp" +"skewSampArgMax" +"skewSampArgMin" +"skewSampArray" +"skewSampDistinct" +"skewSampForEach" +"skewSampIf" +"skewSampMap" +"skewSampMerge" +"skewSampNull" +"skewSampOrDefault" +"skewSampOrNull" +"skewSampResample" +"skewSampSimpleState" +"skewSampState" "sleep" "sleepEachRow" "SMALLINT" @@ -1128,61 +3801,377 @@ "SMALLINT UNSIGNED" "snowflakeToDateTime" "snowflakeToDateTime64" +"soundex" "SOURCE" +"space" "sparkbar" +"sparkBar" +"sparkbarArgMax" +"sparkBarArgMax" +"sparkbarArgMin" +"sparkBarArgMin" +"sparkbarArray" +"sparkBarArray" +"sparkbarDistinct" +"sparkBarDistinct" +"sparkbarForEach" +"sparkBarForEach" +"sparkbarIf" +"sparkBarIf" +"sparkbarMap" +"sparkBarMap" +"sparkbarMerge" +"sparkBarMerge" +"sparkbarNull" +"sparkBarNull" +"sparkbarOrDefault" +"sparkBarOrDefault" +"sparkbarOrNull" +"sparkBarOrNull" +"sparkbarResample" +"sparkBarResample" +"sparkbarSimpleState" +"sparkBarSimpleState" +"sparkbarState" +"sparkBarState" +"SPATIAL" +"splitByAlpha" "splitByChar" "splitByNonAlpha" "splitByRegexp" "splitByString" "splitByWhitespace" +"sqid" +"sqidDecode" +"sqidEncode" +"SQL SECURITY" "SQL_TSI_DAY" "SQL_TSI_HOUR" +"SQL_TSI_MICROSECOND" +"SQL_TSI_MILLISECOND" "SQL_TSI_MINUTE" "SQL_TSI_MONTH" +"SQL_TSI_NANOSECOND" "SQL_TSI_QUARTER" "SQL_TSI_SECOND" "SQL_TSI_WEEK" "SQL_TSI_YEAR" "sqrt" "SS" -"START" +"SSH_KEY" +"SSL_CERTIFICATE" "startsWith" +"startsWithUTF8" "State" +"STATISTIC" +"STD" +"STDArgMax" +"STDArgMin" +"STDArray" "stddevPop" "STDDEV_POP" +"stddevPopArgMax" +"STDDEV_POPArgMax" +"stddevPopArgMin" +"STDDEV_POPArgMin" +"stddevPopArray" +"STDDEV_POPArray" +"stddevPopDistinct" +"STDDEV_POPDistinct" +"stddevPopForEach" +"STDDEV_POPForEach" +"stddevPopIf" +"STDDEV_POPIf" +"stddevPopMap" +"STDDEV_POPMap" +"stddevPopMerge" +"STDDEV_POPMerge" +"stddevPopNull" +"STDDEV_POPNull" +"stddevPopOrDefault" +"STDDEV_POPOrDefault" +"stddevPopOrNull" +"STDDEV_POPOrNull" +"stddevPopResample" +"STDDEV_POPResample" +"stddevPopSimpleState" +"STDDEV_POPSimpleState" "stddevPopStable" +"stddevPopStableArgMax" +"stddevPopStableArgMin" +"stddevPopStableArray" +"stddevPopStableDistinct" +"stddevPopStableForEach" +"stddevPopStableIf" +"stddevPopStableMap" +"stddevPopStableMerge" +"stddevPopStableNull" +"stddevPopStableOrDefault" +"stddevPopStableOrNull" +"stddevPopStableResample" +"stddevPopStableSimpleState" +"stddevPopStableState" +"stddevPopState" +"STDDEV_POPState" "stddevSamp" "STDDEV_SAMP" +"stddevSampArgMax" +"STDDEV_SAMPArgMax" +"stddevSampArgMin" +"STDDEV_SAMPArgMin" +"stddevSampArray" +"STDDEV_SAMPArray" +"stddevSampDistinct" +"STDDEV_SAMPDistinct" +"stddevSampForEach" +"STDDEV_SAMPForEach" +"stddevSampIf" +"STDDEV_SAMPIf" +"stddevSampMap" +"STDDEV_SAMPMap" +"stddevSampMerge" +"STDDEV_SAMPMerge" +"stddevSampNull" +"STDDEV_SAMPNull" +"stddevSampOrDefault" +"STDDEV_SAMPOrDefault" +"stddevSampOrNull" +"STDDEV_SAMPOrNull" +"stddevSampResample" +"STDDEV_SAMPResample" +"stddevSampSimpleState" +"STDDEV_SAMPSimpleState" "stddevSampStable" +"stddevSampStableArgMax" +"stddevSampStableArgMin" +"stddevSampStableArray" +"stddevSampStableDistinct" +"stddevSampStableForEach" +"stddevSampStableIf" +"stddevSampStableMap" +"stddevSampStableMerge" +"stddevSampStableNull" +"stddevSampStableOrDefault" +"stddevSampStableOrNull" +"stddevSampStableResample" +"stddevSampStableSimpleState" +"stddevSampStableState" +"stddevSampState" +"STDDEV_SAMPState" +"STDDistinct" +"STDForEach" +"STDIf" +"STDMap" +"STDMerge" +"STDNull" +"STDOrDefault" +"STDOrNull" +"STDResample" +"STDSimpleState" +"STDState" "stem" "STEP" "stochasticLinearRegression" +"stochasticLinearRegressionArgMax" +"stochasticLinearRegressionArgMin" +"stochasticLinearRegressionArray" +"stochasticLinearRegressionDistinct" +"stochasticLinearRegressionForEach" +"stochasticLinearRegressionIf" +"stochasticLinearRegressionMap" +"stochasticLinearRegressionMerge" +"stochasticLinearRegressionNull" +"stochasticLinearRegressionOrDefault" +"stochasticLinearRegressionOrNull" +"stochasticLinearRegressionResample" +"stochasticLinearRegressionSimpleState" +"stochasticLinearRegressionState" "stochasticLogisticRegression" -"STOP" +"stochasticLogisticRegressionArgMax" +"stochasticLogisticRegressionArgMin" +"stochasticLogisticRegressionArray" +"stochasticLogisticRegressionDistinct" +"stochasticLogisticRegressionForEach" +"stochasticLogisticRegressionIf" +"stochasticLogisticRegressionMap" +"stochasticLogisticRegressionMerge" +"stochasticLogisticRegressionNull" +"stochasticLogisticRegressionOrDefault" +"stochasticLogisticRegressionOrNull" +"stochasticLogisticRegressionResample" +"stochasticLogisticRegressionSimpleState" +"stochasticLogisticRegressionState" +"STORAGE" +"STRICT" +"STRICTLY_ASCENDING" "String" +"stringJaccardIndex" +"stringJaccardIndexUTF8" "stringToH3" +"str_to_date" +"str_to_map" +"structureToCapnProtoSchema" +"structureToProtobufSchema" "studentTTest" +"studentTTestArgMax" +"studentTTestArgMin" +"studentTTestArray" +"studentTTestDistinct" +"studentTTestForEach" +"studentTTestIf" +"studentTTestMap" +"studentTTestMerge" +"studentTTestNull" +"studentTTestOrDefault" +"studentTTestOrNull" +"studentTTestResample" +"studentTTestSimpleState" +"studentTTestState" "subBitmap" +"subDate" +"SUBPARTITION" +"SUBPARTITION BY" +"SUBPARTITIONS" "substr" "substring" "SUBSTRING" +"substringIndex" +"SUBSTRING_INDEX" +"substringIndexUTF8" "substringUTF8" "subtractDays" "subtractHours" +"subtractInterval" +"subtractMicroseconds" +"subtractMilliseconds" "subtractMinutes" "subtractMonths" +"subtractNanoseconds" "subtractQuarters" "subtractSeconds" +"subtractTupleOfIntervals" "subtractWeeks" "subtractYears" "sum" +"sumArgMax" +"sumArgMin" +"sumArray" "sumCount" +"sumCountArgMax" +"sumCountArgMin" +"sumCountArray" +"sumCountDistinct" +"sumCountForEach" +"sumCountIf" +"sumCountMap" +"sumCountMerge" +"sumCountNull" +"sumCountOrDefault" +"sumCountOrNull" +"sumCountResample" +"sumCountSimpleState" +"sumCountState" +"sumDistinct" +"sumForEach" +"sumIf" "sumKahan" +"sumKahanArgMax" +"sumKahanArgMin" +"sumKahanArray" +"sumKahanDistinct" +"sumKahanForEach" +"sumKahanIf" +"sumKahanMap" +"sumKahanMerge" +"sumKahanNull" +"sumKahanOrDefault" +"sumKahanOrNull" +"sumKahanResample" +"sumKahanSimpleState" +"sumKahanState" "sumMap" "sumMapFiltered" +"sumMapFilteredArgMax" +"sumMapFilteredArgMin" +"sumMapFilteredArray" +"sumMapFilteredDistinct" +"sumMapFilteredForEach" +"sumMapFilteredIf" +"sumMapFilteredMap" +"sumMapFilteredMerge" +"sumMapFilteredNull" +"sumMapFilteredOrDefault" +"sumMapFilteredOrNull" +"sumMapFilteredResample" +"sumMapFilteredSimpleState" +"sumMapFilteredState" "sumMapFilteredWithOverflow" +"sumMapFilteredWithOverflowArgMax" +"sumMapFilteredWithOverflowArgMin" +"sumMapFilteredWithOverflowArray" +"sumMapFilteredWithOverflowDistinct" +"sumMapFilteredWithOverflowForEach" +"sumMapFilteredWithOverflowIf" +"sumMapFilteredWithOverflowMap" +"sumMapFilteredWithOverflowMerge" +"sumMapFilteredWithOverflowNull" +"sumMapFilteredWithOverflowOrDefault" +"sumMapFilteredWithOverflowOrNull" +"sumMapFilteredWithOverflowResample" +"sumMapFilteredWithOverflowSimpleState" +"sumMapFilteredWithOverflowState" +"sumMappedArrays" +"sumMappedArraysArgMax" +"sumMappedArraysArgMin" +"sumMappedArraysArray" +"sumMappedArraysDistinct" +"sumMappedArraysForEach" +"sumMappedArraysIf" +"sumMappedArraysMap" +"sumMappedArraysMerge" +"sumMappedArraysNull" +"sumMappedArraysOrDefault" +"sumMappedArraysOrNull" +"sumMappedArraysResample" +"sumMappedArraysSimpleState" +"sumMappedArraysState" "sumMapWithOverflow" +"sumMapWithOverflowArgMax" +"sumMapWithOverflowArgMin" +"sumMapWithOverflowArray" +"sumMapWithOverflowDistinct" +"sumMapWithOverflowForEach" +"sumMapWithOverflowIf" +"sumMapWithOverflowMap" +"sumMapWithOverflowMerge" +"sumMapWithOverflowNull" +"sumMapWithOverflowOrDefault" +"sumMapWithOverflowOrNull" +"sumMapWithOverflowResample" +"sumMapWithOverflowSimpleState" +"sumMapWithOverflowState" +"sumMerge" +"sumNull" +"sumOrDefault" +"sumOrNull" +"sumResample" +"sumSimpleState" +"sumState" "sumWithOverflow" +"sumWithOverflowArgMax" +"sumWithOverflowArgMin" +"sumWithOverflowArray" +"sumWithOverflowDistinct" +"sumWithOverflowForEach" +"sumWithOverflowIf" +"sumWithOverflowMap" +"sumWithOverflowMerge" +"sumWithOverflowNull" +"sumWithOverflowOrDefault" +"sumWithOverflowOrNull" +"sumWithOverflowResample" +"sumWithOverflowSimpleState" +"sumWithOverflowState" "SUSPEND" "svg" "SVG" @@ -1191,24 +4180,44 @@ "SYNTAX" "SYSTEM" "TABLE" +"TABLE OVERRIDE" "TABLES" "tan" "tanh" "tcpPort" "TEMPORARY" +"TEMPORARY TABLE" "TEST" "TEXT" "tgamma" +"theilsU" +"theilsUArgMax" +"theilsUArgMin" +"theilsUArray" +"theilsUDistinct" +"theilsUForEach" +"theilsUIf" +"theilsUMap" +"theilsUMerge" +"theilsUNull" +"theilsUOrDefault" +"theilsUOrNull" +"theilsUResample" +"theilsUSimpleState" +"theilsUState" "THEN" "throwIf" "tid" -"TIES" -"TIMEOUT" +"TIME" +"timeDiff" "timeSlot" "timeSlots" +"timestamp" "TIMESTAMP" "TIMESTAMP_ADD" "TIMESTAMPADD" +"timestamp_diff" +"timestampDiff" "TIMESTAMP_DIFF" "TIMESTAMPDIFF" "TIMESTAMP_SUB" @@ -1226,77 +4235,111 @@ "TINYTEXT" "TO" "TO_BASE64" +"toBool" "toColumnTypeName" "toDate" "toDate32" +"toDate32OrDefault" "toDate32OrNull" "toDate32OrZero" +"toDateOrDefault" "toDateOrNull" "toDateOrZero" "toDateTime" "toDateTime32" "toDateTime64" +"toDateTime64OrDefault" "toDateTime64OrNull" "toDateTime64OrZero" +"toDateTimeOrDefault" "toDateTimeOrNull" "toDateTimeOrZero" "today" "toDayOfMonth" "toDayOfWeek" "toDayOfYear" +"TO_DAYS" +"toDaysSinceYearZero" "toDecimal128" +"toDecimal128OrDefault" "toDecimal128OrNull" "toDecimal128OrZero" "toDecimal256" +"toDecimal256OrDefault" "toDecimal256OrNull" "toDecimal256OrZero" "toDecimal32" +"toDecimal32OrDefault" "toDecimal32OrNull" "toDecimal32OrZero" "toDecimal64" +"toDecimal64OrDefault" "toDecimal64OrNull" "toDecimal64OrZero" +"toDecimalString" "TO DISK" "toFixedString" "toFloat32" +"toFloat32OrDefault" "toFloat32OrNull" "toFloat32OrZero" "toFloat64" +"toFloat64OrDefault" "toFloat64OrNull" "toFloat64OrZero" "toHour" +"TO INNER UUID" "toInt128" +"toInt128OrDefault" "toInt128OrNull" "toInt128OrZero" "toInt16" +"toInt16OrDefault" "toInt16OrNull" "toInt16OrZero" "toInt256" +"toInt256OrDefault" "toInt256OrNull" "toInt256OrZero" "toInt32" +"toInt32OrDefault" "toInt32OrNull" "toInt32OrZero" "toInt64" +"toInt64OrDefault" "toInt64OrNull" "toInt64OrZero" "toInt8" +"toInt8OrDefault" "toInt8OrNull" "toInt8OrZero" "toIntervalDay" "toIntervalHour" +"toIntervalMicrosecond" +"toIntervalMillisecond" "toIntervalMinute" "toIntervalMonth" +"toIntervalNanosecond" "toIntervalQuarter" "toIntervalSecond" "toIntervalWeek" "toIntervalYear" "toIPv4" +"toIPv4OrDefault" +"toIPv4OrNull" +"toIPv4OrZero" "toIPv6" +"toIPv6OrDefault" +"toIPv6OrNull" +"toIPv6OrZero" "toISOWeek" "toISOYear" "toJSONString" +"tokens" +"toLastDayOfMonth" +"toLastDayOfWeek" "toLowCardinality" +"toMillisecond" "toMinute" "toModifiedJulianDay" "toModifiedJulianDayOrNull" @@ -1305,8 +4348,37 @@ "toNullable" "TOP" "topK" +"topKArgMax" +"topKArgMin" +"topKArray" +"topKDistinct" +"topKForEach" +"topKIf" +"topKMap" +"topKMerge" +"topKNull" +"topKOrDefault" +"topKOrNull" +"topKResample" +"topKSimpleState" +"topKState" "topKWeighted" +"topKWeightedArgMax" +"topKWeightedArgMin" +"topKWeightedArray" +"topKWeightedDistinct" +"topKWeightedForEach" +"topKWeightedIf" +"topKWeightedMap" +"topKWeightedMerge" +"topKWeightedNull" +"topKWeightedOrDefault" +"topKWeightedOrNull" +"topKWeightedResample" +"topKWeightedSimpleState" +"topKWeightedState" "topLevelDomain" +"topLevelDomainRFC" "toQuarter" "toRelativeDayNum" "toRelativeHourNum" @@ -1317,14 +4389,19 @@ "toRelativeWeekNum" "toRelativeYearNum" "toSecond" +"TO SHARD" "toStartOfDay" "toStartOfFifteenMinutes" +"toStartOfFiveMinute" "toStartOfFiveMinutes" "toStartOfHour" "toStartOfInterval" "toStartOfISOYear" +"toStartOfMicrosecond" +"toStartOfMillisecond" "toStartOfMinute" "toStartOfMonth" +"toStartOfNanosecond" "toStartOfQuarter" "toStartOfSecond" "toStartOfTenMinutes" @@ -1339,28 +4416,38 @@ "toTimeZone" "toTypeName" "toUInt128" +"toUInt128OrDefault" "toUInt128OrNull" "toUInt128OrZero" "toUInt16" +"toUInt16OrDefault" "toUInt16OrNull" "toUInt16OrZero" "toUInt256" +"toUInt256OrDefault" "toUInt256OrNull" "toUInt256OrZero" "toUInt32" +"toUInt32OrDefault" "toUInt32OrNull" "toUInt32OrZero" "toUInt64" +"toUInt64OrDefault" "toUInt64OrNull" "toUInt64OrZero" "toUInt8" +"toUInt8OrDefault" "toUInt8OrNull" "toUInt8OrZero" +"TO_UNIXTIME" "toUnixTimestamp" "toUnixTimestamp64Micro" "toUnixTimestamp64Milli" "toUnixTimestamp64Nano" +"to_utc_timestamp" +"toUTCTimestamp" "toUUID" +"toUUIDOrDefault" "toUUIDOrNull" "toUUIDOrZero" "toValidUTF8" @@ -1371,23 +4458,55 @@ "toYYYYMM" "toYYYYMMDD" "toYYYYMMDDhhmmss" +"TRACKING ONLY" "TRAILING" +"TRANSACTION" +"transactionID" +"transactionLatestSnapshot" +"transactionOldestSnapshot" "transform" +"translate" +"translateUTF8" +"TRIGGER" +"trim" "TRIM" "trimBoth" "trimLeft" "trimRight" +"TRUE" "trunc" "truncate" "TRUNCATE" +"tryBase58Decode" "tryBase64Decode" +"tryDecrypt" +"tryIdnaEncode" +"tryPunycodeDecode" "TTL" +"tumble" +"tumbleEnd" +"tumbleStart" "tuple" "Tuple" +"tupleConcat" +"tupleDivide" +"tupleDivideByNumber" "tupleElement" "tupleHammingDistance" +"tupleIntDiv" +"tupleIntDivByNumber" +"tupleIntDivOrZero" +"tupleIntDivOrZeroByNumber" +"tupleMinus" +"tupleModulo" +"tupleModuloByNumber" +"tupleMultiply" +"tupleMultiplyByNumber" +"tupleNegate" +"tuplePlus" "tupleToNameValuePairs" "TYPE" +"TYPEOF" "ucase" "UInt128" "UInt16" @@ -1395,41 +4514,245 @@ "UInt32" "UInt64" "UInt8" +"ULIDStringToDateTime" "unbin" +"UNBOUNDED" +"UNDROP" +"UNFREEZE" "unhex" "UNION" "uniq" +"uniqArgMax" +"uniqArgMin" +"uniqArray" "uniqCombined" "uniqCombined64" +"uniqCombined64ArgMax" +"uniqCombined64ArgMin" +"uniqCombined64Array" +"uniqCombined64Distinct" +"uniqCombined64ForEach" +"uniqCombined64If" +"uniqCombined64Map" +"uniqCombined64Merge" +"uniqCombined64Null" +"uniqCombined64OrDefault" +"uniqCombined64OrNull" +"uniqCombined64Resample" +"uniqCombined64SimpleState" +"uniqCombined64State" +"uniqCombinedArgMax" +"uniqCombinedArgMin" +"uniqCombinedArray" +"uniqCombinedDistinct" +"uniqCombinedForEach" +"uniqCombinedIf" +"uniqCombinedMap" +"uniqCombinedMerge" +"uniqCombinedNull" +"uniqCombinedOrDefault" +"uniqCombinedOrNull" +"uniqCombinedResample" +"uniqCombinedSimpleState" +"uniqCombinedState" +"uniqDistinct" "uniqExact" +"uniqExactArgMax" +"uniqExactArgMin" +"uniqExactArray" +"uniqExactDistinct" +"uniqExactForEach" +"uniqExactIf" +"uniqExactMap" +"uniqExactMerge" +"uniqExactNull" +"uniqExactOrDefault" +"uniqExactOrNull" +"uniqExactResample" +"uniqExactSimpleState" +"uniqExactState" +"uniqForEach" "uniqHLL12" +"uniqHLL12ArgMax" +"uniqHLL12ArgMin" +"uniqHLL12Array" +"uniqHLL12Distinct" +"uniqHLL12ForEach" +"uniqHLL12If" +"uniqHLL12Map" +"uniqHLL12Merge" +"uniqHLL12Null" +"uniqHLL12OrDefault" +"uniqHLL12OrNull" +"uniqHLL12Resample" +"uniqHLL12SimpleState" +"uniqHLL12State" +"uniqIf" +"uniqMap" +"uniqMerge" +"uniqNull" +"uniqOrDefault" +"uniqOrNull" +"uniqResample" +"uniqSimpleState" +"uniqState" "uniqTheta" +"uniqThetaArgMax" +"uniqThetaArgMin" +"uniqThetaArray" +"uniqThetaDistinct" +"uniqThetaForEach" +"uniqThetaIf" +"uniqThetaIntersect" +"uniqThetaMap" +"uniqThetaMerge" +"uniqThetaNot" +"uniqThetaNull" +"uniqThetaOrDefault" +"uniqThetaOrNull" +"uniqThetaResample" +"uniqThetaSimpleState" +"uniqThetaState" +"uniqThetaUnion" +"UNIQUE" "uniqUpTo" +"uniqUpToArgMax" +"uniqUpToArgMin" +"uniqUpToArray" +"uniqUpToDistinct" +"uniqUpToForEach" +"uniqUpToIf" +"uniqUpToMap" +"uniqUpToMerge" +"uniqUpToNull" +"uniqUpToOrDefault" +"uniqUpToOrNull" +"uniqUpToResample" +"uniqUpToSimpleState" +"uniqUpToState" +"UNSET FAKE TIME" +"UNSIGNED" "UPDATE" "upper" "upperUTF8" "uptime" +"URL" "URLHash" "URLHierarchy" "URLPathHierarchy" "USE" "user" "USING" +"UTC_timestamp" +"UTCTimestamp" "UUID" "UUIDNumToString" "UUIDStringToNum" "validateNestedArraySizes" +"VALID UNTIL" "VALUES" +"VARBINARY" "VARCHAR" "VARCHAR2" +"Variant" +"variantElement" +"variantType" "varPop" "VAR_POP" +"varPopArgMax" +"VAR_POPArgMax" +"varPopArgMin" +"VAR_POPArgMin" +"varPopArray" +"VAR_POPArray" +"varPopDistinct" +"VAR_POPDistinct" +"varPopForEach" +"VAR_POPForEach" +"varPopIf" +"VAR_POPIf" +"varPopMap" +"VAR_POPMap" +"varPopMerge" +"VAR_POPMerge" +"varPopNull" +"VAR_POPNull" +"varPopOrDefault" +"VAR_POPOrDefault" +"varPopOrNull" +"VAR_POPOrNull" +"varPopResample" +"VAR_POPResample" +"varPopSimpleState" +"VAR_POPSimpleState" "varPopStable" +"varPopStableArgMax" +"varPopStableArgMin" +"varPopStableArray" +"varPopStableDistinct" +"varPopStableForEach" +"varPopStableIf" +"varPopStableMap" +"varPopStableMerge" +"varPopStableNull" +"varPopStableOrDefault" +"varPopStableOrNull" +"varPopStableResample" +"varPopStableSimpleState" +"varPopStableState" +"varPopState" +"VAR_POPState" "varSamp" "VAR_SAMP" +"varSampArgMax" +"VAR_SAMPArgMax" +"varSampArgMin" +"VAR_SAMPArgMin" +"varSampArray" +"VAR_SAMPArray" +"varSampDistinct" +"VAR_SAMPDistinct" +"varSampForEach" +"VAR_SAMPForEach" +"varSampIf" +"VAR_SAMPIf" +"varSampMap" +"VAR_SAMPMap" +"varSampMerge" +"VAR_SAMPMerge" +"varSampNull" +"VAR_SAMPNull" +"varSampOrDefault" +"VAR_SAMPOrDefault" +"varSampOrNull" +"VAR_SAMPOrNull" +"varSampResample" +"VAR_SAMPResample" +"varSampSimpleState" +"VAR_SAMPSimpleState" "varSampStable" +"varSampStableArgMax" +"varSampStableArgMin" +"varSampStableArray" +"varSampStableDistinct" +"varSampStableForEach" +"varSampStableIf" +"varSampStableMap" +"varSampStableMerge" +"varSampStableNull" +"varSampStableOrDefault" +"varSampStableOrNull" +"varSampStableResample" +"varSampStableSimpleState" +"varSampStableState" +"varSampState" +"VAR_SAMPState" +"VARYING" +"vectorDifference" +"vectorSum" "version" "VIEW" +"VISIBLE" "visibleWidth" "visitParamExtractBool" "visitParamExtractFloat" @@ -1438,16 +4761,55 @@ "visitParamExtractString" "visitParamExtractUInt" "visitParamHas" -"VOLUME" "WATCH" +"WATERMARK" "week" "WEEK" +"WEEKS" "welchTTest" +"welchTTestArgMax" +"welchTTestArgMin" +"welchTTestArray" +"welchTTestDistinct" +"welchTTestForEach" +"welchTTestIf" +"welchTTestMap" +"welchTTestMerge" +"welchTTestNull" +"welchTTestOrDefault" +"welchTTestOrNull" +"welchTTestResample" +"welchTTestSimpleState" +"welchTTestState" "WHEN" "WHERE" +"width_bucket" +"widthBucket" +"WINDOW" "windowFunnel" +"windowFunnelArgMax" +"windowFunnelArgMin" +"windowFunnelArray" +"windowFunnelDistinct" +"windowFunnelForEach" +"windowFunnelIf" +"windowFunnelMap" +"windowFunnelMerge" +"windowFunnelNull" +"windowFunnelOrDefault" +"windowFunnelOrNull" +"windowFunnelResample" +"windowFunnelSimpleState" +"windowFunnelState" +"windowID" "WITH" +"WITH ADMIN OPTION" +"WITH CHECK" "WITH FILL" +"WITH GRANT OPTION" +"with_itemindex" +"WITH NAME" +"WITH REPLACE OPTION" "WITH TIES" "WK" "wkt" @@ -1463,14 +4825,23 @@ "wordShingleSimHashCaseInsensitive" "wordShingleSimHashCaseInsensitiveUTF8" "wordShingleSimHashUTF8" +"WRITABLE" "WW" +"wyHash64" "xor" +"xxh3" "xxHash32" "xxHash64" -"kostikConsistentHash" +"yandexConsistentHash" "YEAR" +"YEARS" "yearweek" "yesterday" "YY" "YYYY" +"YYYYMMDDhhmmssToDateTime" +"YYYYMMDDhhmmssToDateTime64" +"YYYYMMDDToDate" +"YYYYMMDDToDate32" +"ZKPATH" "zookeeperSessionUptime" diff --git a/tests/fuzz/dictionaries/datatypes.dict b/tests/fuzz/dictionaries/datatypes.dict index e4983ae5f68..232e89db0c0 100644 --- a/tests/fuzz/dictionaries/datatypes.dict +++ b/tests/fuzz/dictionaries/datatypes.dict @@ -1,185 +1,134 @@ -"Polygon" -"Ring" -"Point" -"SimpleAggregateFunction" -"MultiPolygon" -"IPv6" -"IntervalSecond" +"AggregateFunction" +"Array" +"BIGINT" +"BIGINT SIGNED" +"BIGINT UNSIGNED" +"BINARY" +"BINARY LARGE OBJECT" +"BINARY VARYING" +"BIT" +"BLOB" +"BYTE" +"BYTEA" +"Bool" +"CHAR" +"CHAR LARGE OBJECT" +"CHAR VARYING" +"CHARACTER" +"CHARACTER LARGE OBJECT" +"CHARACTER VARYING" +"CLOB" +"DEC" +"DOUBLE" +"DOUBLE PRECISION" +"Date" +"Date32" +"DateTime" +"DateTime32" +"DateTime64" +"Decimal" +"Decimal128" +"Decimal256" +"Decimal32" +"Decimal64" +"ENUM" +"Enum" +"Enum16" +"Enum8" +"FIXED" +"FLOAT" +"FixedString" +"Float32" +"Float64" +"GEOMETRY" +"INET4" +"INET6" +"INT" +"INT SIGNED" +"INT UNSIGNED" +"INT1" +"INT1 SIGNED" +"INT1 UNSIGNED" +"INTEGER" +"INTEGER SIGNED" +"INTEGER UNSIGNED" "IPv4" -"UInt32" -"IntervalYear" -"IntervalQuarter" -"IntervalMonth" +"IPv6" +"Int128" +"Int16" +"Int256" +"Int32" "Int64" +"Int8" "IntervalDay" "IntervalHour" -"Int16" -"UInt256" -"LowCardinality" -"AggregateFunction" -"Nothing" -"Decimal256" -"Tuple" -"Array" -"Enum16" +"IntervalMicrosecond" +"IntervalMillisecond" "IntervalMinute" -"FixedString" -"String" -"DateTime" -"Map" -"UUID" -"Decimal64" -"Nullable" -"Enum" -"Int32" -"UInt8" -"Date" -"Decimal32" -"UInt128" -"Float64" -"Nested" -"Int128" -"Decimal128" -"Int8" -"Decimal" -"Int256" -"DateTime64" -"Enum8" -"DateTime32" -"Date32" +"IntervalMonth" +"IntervalNanosecond" +"IntervalQuarter" +"IntervalSecond" "IntervalWeek" -"UInt64" -"UInt16" -"Float32" -"INET6" -"INET4" -"ENUM" -"BINARY" -"NATIONAL CHAR VARYING" -"BINARY VARYING" -"NCHAR LARGE OBJECT" -"NATIONAL CHARACTER VARYING" -"NATIONAL CHARACTER LARGE OBJECT" -"NATIONAL CHARACTER" -"NATIONAL CHAR" -"CHARACTER VARYING" +"IntervalYear" +"JSON" "LONGBLOB" -"MEDIUMTEXT" -"TEXT" -"TINYBLOB" -"VARCHAR2" -"CHARACTER LARGE OBJECT" -"DOUBLE PRECISION" "LONGTEXT" -"NVARCHAR" -"INT1 UNSIGNED" -"VARCHAR" -"CHAR VARYING" +"LowCardinality" "MEDIUMBLOB" -"NCHAR" -"CHAR" -"SMALLINT UNSIGNED" -"TIMESTAMP" -"FIXED" -"TINYTEXT" -"NUMERIC" -"DEC" -"TINYINT UNSIGNED" -"INTEGER UNSIGNED" -"INT UNSIGNED" -"CLOB" -"MEDIUMINT UNSIGNED" -"BOOL" -"SMALLINT" -"INTEGER SIGNED" -"NCHAR VARYING" -"INT SIGNED" -"TINYINT SIGNED" -"BIGINT SIGNED" -"BINARY LARGE OBJECT" -"SMALLINT SIGNED" "MEDIUMINT" -"INTEGER" -"INT1 SIGNED" -"BIGINT UNSIGNED" -"BYTEA" -"INT" -"SINGLE" -"FLOAT" "MEDIUMINT SIGNED" -"BOOLEAN" -"DOUBLE" -"INT1" -"CHAR LARGE OBJECT" -"TINYINT" -"BIGINT" -"CHARACTER" -"BYTE" -"BLOB" +"MEDIUMINT UNSIGNED" +"MEDIUMTEXT" +"Map" +"MultiPolygon" +"NATIONAL CHAR" +"NATIONAL CHAR VARYING" +"NATIONAL CHARACTER" +"NATIONAL CHARACTER LARGE OBJECT" +"NATIONAL CHARACTER VARYING" +"NCHAR" +"NCHAR LARGE OBJECT" +"NCHAR VARYING" +"NUMERIC" +"NVARCHAR" +"Nested" +"Nothing" +"Nullable" +"Object" +"Point" +"Polygon" "REAL" -"IPv6" -"IPv4" -"Enum" -"FixedString" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"String" -"Float64" -"String" -"String" -"UInt8" -"String" -"String" -"String" -"String" +"Ring" +"SET" +"SIGNED" +"SINGLE" +"SMALLINT" +"SMALLINT SIGNED" +"SMALLINT UNSIGNED" +"SimpleAggregateFunction" "String" +"TEXT" +"TIME" +"TIMESTAMP" +"TINYBLOB" +"TINYINT" +"TINYINT SIGNED" +"TINYINT UNSIGNED" +"TINYTEXT" +"Tuple" +"UInt128" "UInt16" -"DateTime" -"Decimal" -"String" -"Decimal" -"Decimal" -"UInt8" +"UInt256" "UInt32" -"UInt32" -"String" -"UInt32" -"Int8" -"Int16" -"Int32" -"String" -"Int32" -"Int8" -"Int64" -"String" -"Int16" -"Int32" -"Int32" -"Int8" "UInt64" -"String" -"Int32" -"Float32" -"Float32" -"Int32" -"Int8" -"Float64" -"Int8" -"String" -"Int8" -"Int64" -"String" -"Int8" -"String" -"Float32" +"UInt8" +"UNSIGNED" +"UUID" +"VARBINARY" +"VARCHAR" +"VARCHAR2" +"Variant" +"YEAR" +"bool" +"boolean" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index 3fe8dbfe501..ec7f8017fb2 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -1,1134 +1,4172 @@ -"logTrace" -"aes_decrypt_mysql" -"aes_encrypt_mysql" -"decrypt" -"encrypt" -"bayesAB" -"snowflakeToDateTime64" -"snowflakeToDateTime" -"validateNestedArraySizes" -"wordShingleMinHashArgCaseInsensitiveUTF8" -"wordShingleMinHashArgUTF8" -"wordShingleMinHashArgCaseInsensitive" -"wordShingleMinHashArg" -"ngramMinHashArgCaseInsensitiveUTF8" -"ngramMinHashArgUTF8" -"ngramMinHashArgCaseInsensitive" -"wordShingleMinHashCaseInsensitiveUTF8" -"wordShingleMinHashUTF8" -"ngramMinHashCaseInsensitiveUTF8" -"ngramMinHashUTF8" -"ngramMinHashCaseInsensitive" -"wordShingleSimHashUTF8" -"ngramSimHashUTF8" -"ngramSimHashCaseInsensitive" -"ngramSimHash" -"toUnixTimestamp64Nano" -"toUnixTimestamp64Micro" -"jumpConsistentHash" -"kostikConsistentHash" -"addressToSymbol" -"toJSONString" -"JSON_VALUE" -"JSONExtractKeysAndValuesRaw" +"BIT_AND" +"BIT_ANDArgMax" +"BIT_ANDArgMin" +"BIT_ANDArray" +"BIT_ANDDistinct" +"BIT_ANDForEach" +"BIT_ANDIf" +"BIT_ANDMap" +"BIT_ANDMerge" +"BIT_ANDNull" +"BIT_ANDOrDefault" +"BIT_ANDOrNull" +"BIT_ANDResample" +"BIT_ANDSimpleState" +"BIT_ANDState" +"BIT_OR" +"BIT_ORArgMax" +"BIT_ORArgMin" +"BIT_ORArray" +"BIT_ORDistinct" +"BIT_ORForEach" +"BIT_ORIf" +"BIT_ORMap" +"BIT_ORMerge" +"BIT_ORNull" +"BIT_OROrDefault" +"BIT_OROrNull" +"BIT_ORResample" +"BIT_ORSimpleState" +"BIT_ORState" +"BIT_XOR" +"BIT_XORArgMax" +"BIT_XORArgMin" +"BIT_XORArray" +"BIT_XORDistinct" +"BIT_XORForEach" +"BIT_XORIf" +"BIT_XORMap" +"BIT_XORMerge" +"BIT_XORNull" +"BIT_XOROrDefault" +"BIT_XOROrNull" +"BIT_XORResample" +"BIT_XORSimpleState" +"BIT_XORState" +"BLAKE3" +"CAST" +"CHARACTER_LENGTH" +"CHAR_LENGTH" +"COVAR_POP" +"COVAR_POPArgMax" +"COVAR_POPArgMin" +"COVAR_POPArray" +"COVAR_POPDistinct" +"COVAR_POPForEach" +"COVAR_POPIf" +"COVAR_POPMap" +"COVAR_POPMerge" +"COVAR_POPNull" +"COVAR_POPOrDefault" +"COVAR_POPOrNull" +"COVAR_POPResample" +"COVAR_POPSimpleState" +"COVAR_POPState" +"COVAR_SAMP" +"COVAR_SAMPArgMax" +"COVAR_SAMPArgMin" +"COVAR_SAMPArray" +"COVAR_SAMPDistinct" +"COVAR_SAMPForEach" +"COVAR_SAMPIf" +"COVAR_SAMPMap" +"COVAR_SAMPMerge" +"COVAR_SAMPNull" +"COVAR_SAMPOrDefault" +"COVAR_SAMPOrNull" +"COVAR_SAMPResample" +"COVAR_SAMPSimpleState" +"COVAR_SAMPState" +"CRC32" +"CRC32IEEE" +"CRC64" +"DATABASE" +"DATE" +"DATE_DIFF" +"DATE_FORMAT" +"DATE_TRUNC" +"DAY" +"DAYOFMONTH" +"DAYOFWEEK" +"DAYOFYEAR" +"FORMAT_BYTES" +"FQDN" +"FROM_BASE64" +"FROM_DAYS" +"FROM_UNIXTIME" +"HOUR" +"INET6_ATON" +"INET6_NTOA" +"INET_ATON" +"INET_NTOA" +"IPv4CIDRToRange" +"IPv4NumToString" +"IPv4NumToStringClassC" +"IPv4StringToNum" +"IPv4StringToNumOrDefault" +"IPv4StringToNumOrNull" +"IPv4ToIPv6" +"IPv6CIDRToRange" +"IPv6NumToString" +"IPv6StringToNum" +"IPv6StringToNumOrDefault" +"IPv6StringToNumOrNull" +"JSONArrayLength" +"JSONExtract" +"JSONExtractArrayRaw" +"JSONExtractBool" +"JSONExtractFloat" +"JSONExtractInt" "JSONExtractKeys" +"JSONExtractKeysAndValues" +"JSONExtractKeysAndValuesRaw" +"JSONExtractRaw" "JSONExtractString" -"JSONType" +"JSONExtractUInt" +"JSONHas" "JSONKey" "JSONLength" -"isValidJSON" -"isZeroOrNull" -"assumeNotNull" -"s2CapUnion" -"s2CapContains" -"s2CellsIntersect" -"s2GetNeighbors" -"s2ToGeo" -"h3GetFaces" -"h3IsResClassIII" -"h3ToString" -"stringToH3" -"h3ToParent" -"h3GetResolution" -"h3EdgeLengthM" -"svg" -"SVG" -"equals" -"geohashesInBox" -"polygonsIntersectionCartesian" -"polygonPerimeterSpherical" -"bitHammingDistance" -"polygonsDistanceSpherical" -"polygonsSymDifferenceSpherical" -"polygonAreaSpherical" -"greatCircleDistance" -"toInt256" -"mapKeys" -"sign" -"reinterpretAsUInt8" -"atanh" -"formatReadableTimeDelta" -"parseTimeDelta" -"geohashEncode" -"atan2" -"acos" +"JSONType" +"JSON_ARRAY_LENGTH" +"JSON_EXISTS" +"JSON_QUERY" +"JSON_VALUE" +"L1Distance" +"L1Norm" +"L1Normalize" +"L2Distance" +"L2Norm" +"L2Normalize" +"L2SquaredDistance" +"L2SquaredNorm" +"LAST_DAY" +"LinfDistance" +"LinfNorm" +"LinfNormalize" +"LpDistance" +"LpNorm" +"LpNormalize" +"MACNumToString" +"MACStringToNum" +"MACStringToOUI" +"MAP_FROM_ARRAYS" +"MD4" +"MD5" +"MILLISECOND" +"MINUTE" +"MONTH" +"OCTET_LENGTH" +"QUARTER" +"REGEXP_EXTRACT" +"REGEXP_MATCHES" +"REGEXP_REPLACE" +"SCHEMA" +"SECOND" +"SHA1" "SHA224" -"visitParamExtractBool" -"h3kRing" -"asin" -"sin" -"reinterpretAsUInt128" -"lgamma" -"log2" -"geoToH3" -"toUnixTimestamp64Milli" -"firstSignificantSubdomainCustom" -"simpleJSONExtractBool" -"visitParamExtractFloat" -"visitParamExtractInt" -"simpleJSONHas" -"visitParamHas" -"sigmoid" -"cutQueryStringAndFragment" -"dictGetInt16OrDefault" -"cutToFirstSignificantSubdomainCustom" +"SHA256" +"SHA384" +"SHA512" +"SHA512_256" +"STD" +"STDArgMax" +"STDArgMin" +"STDArray" +"STDDEV_POP" +"STDDEV_POPArgMax" +"STDDEV_POPArgMin" +"STDDEV_POPArray" +"STDDEV_POPDistinct" +"STDDEV_POPForEach" +"STDDEV_POPIf" +"STDDEV_POPMap" +"STDDEV_POPMerge" +"STDDEV_POPNull" +"STDDEV_POPOrDefault" +"STDDEV_POPOrNull" +"STDDEV_POPResample" +"STDDEV_POPSimpleState" +"STDDEV_POPState" +"STDDEV_SAMP" +"STDDEV_SAMPArgMax" +"STDDEV_SAMPArgMin" +"STDDEV_SAMPArray" +"STDDEV_SAMPDistinct" +"STDDEV_SAMPForEach" +"STDDEV_SAMPIf" +"STDDEV_SAMPMap" +"STDDEV_SAMPMerge" +"STDDEV_SAMPNull" +"STDDEV_SAMPOrDefault" +"STDDEV_SAMPOrNull" +"STDDEV_SAMPResample" +"STDDEV_SAMPSimpleState" +"STDDEV_SAMPState" +"STDDistinct" +"STDForEach" +"STDIf" +"STDMap" +"STDMerge" +"STDNull" +"STDOrDefault" +"STDOrNull" +"STDResample" +"STDSimpleState" +"STDState" +"SUBSTRING_INDEX" +"SVG" +"TIMESTAMP_DIFF" +"TO_BASE64" +"TO_DAYS" +"TO_UNIXTIME" +"ULIDStringToDateTime" +"URLHash" "URLHierarchy" -"extractURLParameterNames" -"toDecimal128OrZero" -"extractURLParameters" -"path" -"pathFull" -"port" -"domain" +"URLPathHierarchy" +"UTCTimestamp" +"UTC_timestamp" +"UUIDNumToString" +"UUIDStringToNum" +"VAR_POP" +"VAR_POPArgMax" +"VAR_POPArgMin" +"VAR_POPArray" +"VAR_POPDistinct" +"VAR_POPForEach" +"VAR_POPIf" +"VAR_POPMap" +"VAR_POPMerge" +"VAR_POPNull" +"VAR_POPOrDefault" +"VAR_POPOrNull" +"VAR_POPResample" +"VAR_POPSimpleState" +"VAR_POPState" +"VAR_SAMP" +"VAR_SAMPArgMax" +"VAR_SAMPArgMin" +"VAR_SAMPArray" +"VAR_SAMPDistinct" +"VAR_SAMPForEach" +"VAR_SAMPIf" +"VAR_SAMPMap" +"VAR_SAMPMerge" +"VAR_SAMPNull" +"VAR_SAMPOrDefault" +"VAR_SAMPOrNull" +"VAR_SAMPResample" +"VAR_SAMPSimpleState" +"VAR_SAMPState" +"YEAR" +"YYYYMMDDToDate" +"YYYYMMDDToDate32" +"YYYYMMDDhhmmssToDateTime" +"YYYYMMDDhhmmssToDateTime64" +"_CAST" +"__bitBoolMaskAnd" +"__bitBoolMaskOr" +"__bitSwapLastTwo" +"__bitWrapperFunc" +"__getScalar" +"__scalarSubqueryResult" +"abs" +"accurateCast" +"accurateCastOrDefault" +"accurateCastOrNull" +"acos" +"acosh" +"addDate" +"addDays" +"addHours" +"addInterval" +"addMicroseconds" +"addMilliseconds" +"addMinutes" +"addMonths" +"addNanoseconds" +"addQuarters" +"addSeconds" +"addTupleOfIntervals" +"addWeeks" +"addYears" +"addressToLine" +"addressToLineWithInlines" +"addressToSymbol" +"aes_decrypt_mysql" +"aes_encrypt_mysql" +"age" +"aggThrow" +"aggThrowArgMax" +"aggThrowArgMin" +"aggThrowArray" +"aggThrowDistinct" +"aggThrowForEach" +"aggThrowIf" +"aggThrowMap" +"aggThrowMerge" +"aggThrowNull" +"aggThrowOrDefault" +"aggThrowOrNull" +"aggThrowResample" +"aggThrowSimpleState" +"aggThrowState" +"alphaTokens" +"analysisOfVariance" +"analysisOfVarianceArgMax" +"analysisOfVarianceArgMin" +"analysisOfVarianceArray" +"analysisOfVarianceDistinct" +"analysisOfVarianceForEach" +"analysisOfVarianceIf" +"analysisOfVarianceMap" +"analysisOfVarianceMerge" +"analysisOfVarianceNull" +"analysisOfVarianceOrDefault" +"analysisOfVarianceOrNull" +"analysisOfVarianceResample" +"analysisOfVarianceSimpleState" +"analysisOfVarianceState" +"and" +"anova" +"anovaArgMax" +"anovaArgMin" +"anovaArray" +"anovaDistinct" +"anovaForEach" +"anovaIf" +"anovaMap" +"anovaMerge" +"anovaNull" +"anovaOrDefault" +"anovaOrNull" +"anovaResample" +"anovaSimpleState" +"anovaState" +"any" +"anyArgMax" +"anyArgMin" +"anyArray" +"anyDistinct" +"anyForEach" +"anyHeavy" +"anyHeavyArgMax" +"anyHeavyArgMin" +"anyHeavyArray" +"anyHeavyDistinct" +"anyHeavyForEach" +"anyHeavyIf" +"anyHeavyMap" +"anyHeavyMerge" +"anyHeavyNull" +"anyHeavyOrDefault" +"anyHeavyOrNull" +"anyHeavyResample" +"anyHeavySimpleState" +"anyHeavyState" +"anyIf" +"anyLast" +"anyLastArgMax" +"anyLastArgMin" +"anyLastArray" +"anyLastDistinct" +"anyLastForEach" +"anyLastIf" +"anyLastMap" +"anyLastMerge" +"anyLastNull" +"anyLastOrDefault" +"anyLastOrNull" +"anyLastResample" +"anyLastSimpleState" +"anyLastState" +"anyLast_respect_nulls" +"anyLast_respect_nullsArgMax" +"anyLast_respect_nullsArgMin" +"anyLast_respect_nullsArray" +"anyLast_respect_nullsDistinct" +"anyLast_respect_nullsForEach" +"anyLast_respect_nullsIf" +"anyLast_respect_nullsMap" +"anyLast_respect_nullsMerge" +"anyLast_respect_nullsNull" +"anyLast_respect_nullsOrDefault" +"anyLast_respect_nullsOrNull" +"anyLast_respect_nullsResample" +"anyLast_respect_nullsSimpleState" +"anyLast_respect_nullsState" +"anyMap" +"anyMerge" +"anyNull" +"anyOrDefault" +"anyOrNull" +"anyResample" +"anySimpleState" +"anyState" +"any_respect_nulls" +"any_respect_nullsArgMax" +"any_respect_nullsArgMin" +"any_respect_nullsArray" +"any_respect_nullsDistinct" +"any_respect_nullsForEach" +"any_respect_nullsIf" +"any_respect_nullsMap" +"any_respect_nullsMerge" +"any_respect_nullsNull" +"any_respect_nullsOrDefault" +"any_respect_nullsOrNull" +"any_respect_nullsResample" +"any_respect_nullsSimpleState" +"any_respect_nullsState" +"any_value" +"any_valueArgMax" +"any_valueArgMin" +"any_valueArray" +"any_valueDistinct" +"any_valueForEach" +"any_valueIf" +"any_valueMap" +"any_valueMerge" +"any_valueNull" +"any_valueOrDefault" +"any_valueOrNull" +"any_valueResample" +"any_valueSimpleState" +"any_valueState" +"any_value_respect_nulls" +"any_value_respect_nullsArgMax" +"any_value_respect_nullsArgMin" +"any_value_respect_nullsArray" +"any_value_respect_nullsDistinct" +"any_value_respect_nullsForEach" +"any_value_respect_nullsIf" +"any_value_respect_nullsMap" +"any_value_respect_nullsMerge" +"any_value_respect_nullsNull" +"any_value_respect_nullsOrDefault" +"any_value_respect_nullsOrNull" +"any_value_respect_nullsResample" +"any_value_respect_nullsSimpleState" +"any_value_respect_nullsState" +"appendTrailingCharIfAbsent" +"approx_top_count" +"approx_top_countArgMax" +"approx_top_countArgMin" +"approx_top_countArray" +"approx_top_countDistinct" +"approx_top_countForEach" +"approx_top_countIf" +"approx_top_countMap" +"approx_top_countMerge" +"approx_top_countNull" +"approx_top_countOrDefault" +"approx_top_countOrNull" +"approx_top_countResample" +"approx_top_countSimpleState" +"approx_top_countState" +"approx_top_k" +"approx_top_kArgMax" +"approx_top_kArgMin" +"approx_top_kArray" +"approx_top_kDistinct" +"approx_top_kForEach" +"approx_top_kIf" +"approx_top_kMap" +"approx_top_kMerge" +"approx_top_kNull" +"approx_top_kOrDefault" +"approx_top_kOrNull" +"approx_top_kResample" +"approx_top_kSimpleState" +"approx_top_kState" +"approx_top_sum" +"approx_top_sumArgMax" +"approx_top_sumArgMin" +"approx_top_sumArray" +"approx_top_sumDistinct" +"approx_top_sumForEach" +"approx_top_sumIf" +"approx_top_sumMap" +"approx_top_sumMerge" +"approx_top_sumNull" +"approx_top_sumOrDefault" +"approx_top_sumOrNull" +"approx_top_sumResample" +"approx_top_sumSimpleState" +"approx_top_sumState" +"argMax" +"argMaxArgMax" +"argMaxArgMin" +"argMaxArray" +"argMaxDistinct" +"argMaxForEach" +"argMaxIf" +"argMaxMap" +"argMaxMerge" +"argMaxNull" +"argMaxOrDefault" +"argMaxOrNull" +"argMaxResample" +"argMaxSimpleState" +"argMaxState" +"argMin" +"argMinArgMax" +"argMinArgMin" +"argMinArray" +"argMinDistinct" +"argMinForEach" +"argMinIf" +"argMinMap" +"argMinMerge" +"argMinNull" +"argMinOrDefault" +"argMinOrNull" +"argMinResample" +"argMinSimpleState" +"argMinState" +"array" +"arrayAUC" +"arrayAll" +"arrayAvg" +"arrayCompact" +"arrayConcat" +"arrayCount" +"arrayCumSum" +"arrayCumSumNonNegative" +"arrayDifference" +"arrayDistinct" +"arrayDotProduct" +"arrayElement" +"arrayEnumerate" +"arrayEnumerateDense" +"arrayEnumerateDenseRanked" +"arrayEnumerateUniq" +"arrayEnumerateUniqRanked" +"arrayExists" +"arrayFill" +"arrayFilter" +"arrayFirst" +"arrayFirstIndex" +"arrayFirstOrNull" +"arrayFlatten" +"arrayFold" +"arrayIntersect" +"arrayJaccardIndex" +"arrayJoin" +"arrayLast" +"arrayLastIndex" +"arrayLastOrNull" +"arrayMap" +"arrayMax" +"arrayMin" +"arrayPartialReverseSort" +"arrayPartialShuffle" +"arrayPartialSort" +"arrayPopBack" +"arrayPopFront" +"arrayProduct" +"arrayPushBack" +"arrayPushFront" +"arrayRandomSample" +"arrayReduce" +"arrayReduceInRanges" +"arrayResize" +"arrayReverse" +"arrayReverseFill" +"arrayReverseSort" +"arrayReverseSplit" +"arrayRotateLeft" +"arrayRotateRight" +"arrayShiftLeft" +"arrayShiftRight" +"arrayShingles" +"arrayShuffle" +"arraySlice" +"arraySort" +"arraySplit" +"arrayStringConcat" +"arraySum" +"arrayUniq" +"arrayWithConstant" +"arrayZip" +"array_agg" +"array_aggArgMax" +"array_aggArgMin" +"array_aggArray" +"array_aggDistinct" +"array_aggForEach" +"array_aggIf" +"array_aggMap" +"array_aggMerge" +"array_aggNull" +"array_aggOrDefault" +"array_aggOrNull" +"array_aggResample" +"array_aggSimpleState" +"array_aggState" +"array_concat_agg" +"array_concat_aggArgMax" +"array_concat_aggArgMin" +"array_concat_aggArray" +"array_concat_aggDistinct" +"array_concat_aggForEach" +"array_concat_aggIf" +"array_concat_aggMap" +"array_concat_aggMerge" +"array_concat_aggNull" +"array_concat_aggOrDefault" +"array_concat_aggOrNull" +"array_concat_aggResample" +"array_concat_aggSimpleState" +"array_concat_aggState" +"ascii" +"asin" +"asinh" +"assumeNotNull" +"atan" +"atan2" +"atanh" +"avg" +"avgArgMax" +"avgArgMin" +"avgArray" +"avgDistinct" +"avgForEach" +"avgIf" +"avgMap" +"avgMerge" +"avgNull" +"avgOrDefault" +"avgOrNull" +"avgResample" +"avgSimpleState" +"avgState" +"avgWeighted" +"avgWeightedArgMax" +"avgWeightedArgMin" +"avgWeightedArray" +"avgWeightedDistinct" +"avgWeightedForEach" +"avgWeightedIf" +"avgWeightedMap" +"avgWeightedMerge" +"avgWeightedNull" +"avgWeightedOrDefault" +"avgWeightedOrNull" +"avgWeightedResample" +"avgWeightedSimpleState" +"avgWeightedState" +"bar" +"base58Decode" +"base58Encode" +"base64Decode" +"base64Encode" +"basename" +"bin" +"bitAnd" +"bitCount" +"bitHammingDistance" +"bitNot" +"bitOr" +"bitPositionsToArray" +"bitRotateLeft" +"bitRotateRight" +"bitShiftLeft" +"bitShiftRight" +"bitSlice" "bitTest" -"ngramSearchCaseInsensitiveUTF8" +"bitTestAll" +"bitTestAny" +"bitXor" +"bitmapAnd" +"bitmapAndCardinality" +"bitmapAndnot" +"bitmapAndnotCardinality" +"bitmapBuild" +"bitmapCardinality" +"bitmapContains" +"bitmapHasAll" +"bitmapHasAny" "bitmapMax" -"ngramDistanceCaseInsensitiveUTF8" -"ngramDistance" +"bitmapMin" +"bitmapOr" +"bitmapOrCardinality" +"bitmapSubsetInRange" +"bitmapSubsetLimit" +"bitmapToArray" +"bitmapTransform" +"bitmapXor" +"bitmapXorCardinality" +"bitmaskToArray" +"bitmaskToList" +"blockNumber" +"blockSerializedSize" +"blockSize" +"boundingRatio" +"boundingRatioArgMax" +"boundingRatioArgMin" +"boundingRatioArray" +"boundingRatioDistinct" +"boundingRatioForEach" +"boundingRatioIf" +"boundingRatioMap" +"boundingRatioMerge" +"boundingRatioNull" +"boundingRatioOrDefault" +"boundingRatioOrNull" +"boundingRatioResample" +"boundingRatioSimpleState" +"boundingRatioState" +"buildId" +"byteHammingDistance" +"byteSize" +"byteSlice" +"byteSwap" +"caseWithExpr" +"caseWithExpression" +"caseWithoutExpr" +"caseWithoutExpression" +"catboostEvaluate" +"categoricalInformationValue" +"categoricalInformationValueArgMax" +"categoricalInformationValueArgMin" +"categoricalInformationValueArray" +"categoricalInformationValueDistinct" +"categoricalInformationValueForEach" +"categoricalInformationValueIf" +"categoricalInformationValueMap" +"categoricalInformationValueMerge" +"categoricalInformationValueNull" +"categoricalInformationValueOrDefault" +"categoricalInformationValueOrNull" +"categoricalInformationValueResample" +"categoricalInformationValueSimpleState" +"categoricalInformationValueState" +"cbrt" +"ceil" +"ceiling" +"char" +"cityHash64" +"coalesce" +"concat" +"concatAssumeInjective" +"concatWithSeparator" +"concatWithSeparatorAssumeInjective" +"concat_ws" +"connectionId" +"connection_id" +"contingency" +"contingencyArgMax" +"contingencyArgMin" +"contingencyArray" +"contingencyDistinct" +"contingencyForEach" +"contingencyIf" +"contingencyMap" +"contingencyMerge" +"contingencyNull" +"contingencyOrDefault" +"contingencyOrNull" +"contingencyResample" +"contingencySimpleState" +"contingencyState" +"convertCharset" +"corr" +"corrArgMax" +"corrArgMin" +"corrArray" +"corrDistinct" +"corrForEach" +"corrIf" +"corrMap" +"corrMatrix" +"corrMatrixArgMax" +"corrMatrixArgMin" +"corrMatrixArray" +"corrMatrixDistinct" +"corrMatrixForEach" +"corrMatrixIf" +"corrMatrixMap" +"corrMatrixMerge" +"corrMatrixNull" +"corrMatrixOrDefault" +"corrMatrixOrNull" +"corrMatrixResample" +"corrMatrixSimpleState" +"corrMatrixState" +"corrMerge" +"corrNull" +"corrOrDefault" +"corrOrNull" +"corrResample" +"corrSimpleState" +"corrStable" +"corrStableArgMax" +"corrStableArgMin" +"corrStableArray" +"corrStableDistinct" +"corrStableForEach" +"corrStableIf" +"corrStableMap" +"corrStableMerge" +"corrStableNull" +"corrStableOrDefault" +"corrStableOrNull" +"corrStableResample" +"corrStableSimpleState" +"corrStableState" +"corrState" +"cos" +"cosh" +"cosineDistance" +"count" +"countArgMax" +"countArgMin" +"countArray" +"countDigits" +"countDistinct" +"countEqual" +"countForEach" +"countIf" +"countMap" +"countMatches" +"countMatchesCaseInsensitive" +"countMerge" +"countNull" +"countOrDefault" +"countOrNull" +"countResample" +"countSimpleState" +"countState" +"countSubstrings" +"countSubstringsCaseInsensitive" +"countSubstringsCaseInsensitiveUTF8" +"covarPop" +"covarPopArgMax" +"covarPopArgMin" +"covarPopArray" +"covarPopDistinct" +"covarPopForEach" +"covarPopIf" +"covarPopMap" +"covarPopMatrix" +"covarPopMatrixArgMax" +"covarPopMatrixArgMin" +"covarPopMatrixArray" +"covarPopMatrixDistinct" +"covarPopMatrixForEach" +"covarPopMatrixIf" +"covarPopMatrixMap" +"covarPopMatrixMerge" +"covarPopMatrixNull" +"covarPopMatrixOrDefault" +"covarPopMatrixOrNull" +"covarPopMatrixResample" +"covarPopMatrixSimpleState" +"covarPopMatrixState" +"covarPopMerge" +"covarPopNull" +"covarPopOrDefault" +"covarPopOrNull" +"covarPopResample" +"covarPopSimpleState" +"covarPopStable" +"covarPopStableArgMax" +"covarPopStableArgMin" +"covarPopStableArray" +"covarPopStableDistinct" +"covarPopStableForEach" +"covarPopStableIf" +"covarPopStableMap" +"covarPopStableMerge" +"covarPopStableNull" +"covarPopStableOrDefault" +"covarPopStableOrNull" +"covarPopStableResample" +"covarPopStableSimpleState" +"covarPopStableState" +"covarPopState" +"covarSamp" +"covarSampArgMax" +"covarSampArgMin" +"covarSampArray" +"covarSampDistinct" +"covarSampForEach" +"covarSampIf" +"covarSampMap" +"covarSampMatrix" +"covarSampMatrixArgMax" +"covarSampMatrixArgMin" +"covarSampMatrixArray" +"covarSampMatrixDistinct" +"covarSampMatrixForEach" +"covarSampMatrixIf" +"covarSampMatrixMap" +"covarSampMatrixMerge" +"covarSampMatrixNull" +"covarSampMatrixOrDefault" +"covarSampMatrixOrNull" +"covarSampMatrixResample" +"covarSampMatrixSimpleState" +"covarSampMatrixState" +"covarSampMerge" +"covarSampNull" +"covarSampOrDefault" +"covarSampOrNull" +"covarSampResample" +"covarSampSimpleState" +"covarSampStable" +"covarSampStableArgMax" +"covarSampStableArgMin" +"covarSampStableArray" +"covarSampStableDistinct" +"covarSampStableForEach" +"covarSampStableIf" +"covarSampStableMap" +"covarSampStableMerge" +"covarSampStableNull" +"covarSampStableOrDefault" +"covarSampStableOrNull" +"covarSampStableResample" +"covarSampStableSimpleState" +"covarSampStableState" +"covarSampState" +"cramersV" +"cramersVArgMax" +"cramersVArgMin" +"cramersVArray" +"cramersVBiasCorrected" +"cramersVBiasCorrectedArgMax" +"cramersVBiasCorrectedArgMin" +"cramersVBiasCorrectedArray" +"cramersVBiasCorrectedDistinct" +"cramersVBiasCorrectedForEach" +"cramersVBiasCorrectedIf" +"cramersVBiasCorrectedMap" +"cramersVBiasCorrectedMerge" +"cramersVBiasCorrectedNull" +"cramersVBiasCorrectedOrDefault" +"cramersVBiasCorrectedOrNull" +"cramersVBiasCorrectedResample" +"cramersVBiasCorrectedSimpleState" +"cramersVBiasCorrectedState" +"cramersVDistinct" +"cramersVForEach" +"cramersVIf" +"cramersVMap" +"cramersVMerge" +"cramersVNull" +"cramersVOrDefault" +"cramersVOrNull" +"cramersVResample" +"cramersVSimpleState" +"cramersVState" +"curdate" +"currentDatabase" +"currentProfiles" +"currentRoles" +"currentSchemas" +"currentUser" +"current_database" +"current_date" +"current_schemas" +"current_timestamp" +"cutFragment" +"cutIPv6" +"cutQueryString" +"cutQueryStringAndFragment" +"cutToFirstSignificantSubdomain" +"cutToFirstSignificantSubdomainCustom" +"cutToFirstSignificantSubdomainCustomRFC" +"cutToFirstSignificantSubdomainCustomWithWWW" +"cutToFirstSignificantSubdomainCustomWithWWWRFC" +"cutToFirstSignificantSubdomainRFC" +"cutToFirstSignificantSubdomainWithWWW" +"cutToFirstSignificantSubdomainWithWWWRFC" +"cutURLParameter" +"cutWWW" +"damerauLevenshteinDistance" +"dateDiff" +"dateName" +"dateTime64ToSnowflake" +"dateTimeToSnowflake" +"dateTrunc" +"date_diff" +"decodeHTMLComponent" +"decodeURLComponent" +"decodeURLFormComponent" +"decodeXMLComponent" +"decrypt" +"defaultProfiles" +"defaultRoles" +"defaultValueOfArgumentType" +"defaultValueOfTypeName" +"degrees" +"deltaSum" +"deltaSumArgMax" +"deltaSumArgMin" +"deltaSumArray" +"deltaSumDistinct" +"deltaSumForEach" +"deltaSumIf" +"deltaSumMap" +"deltaSumMerge" +"deltaSumNull" +"deltaSumOrDefault" +"deltaSumOrNull" +"deltaSumResample" +"deltaSumSimpleState" +"deltaSumState" +"deltaSumTimestamp" +"deltaSumTimestampArgMax" +"deltaSumTimestampArgMin" +"deltaSumTimestampArray" +"deltaSumTimestampDistinct" +"deltaSumTimestampForEach" +"deltaSumTimestampIf" +"deltaSumTimestampMap" +"deltaSumTimestampMerge" +"deltaSumTimestampNull" +"deltaSumTimestampOrDefault" +"deltaSumTimestampOrNull" +"deltaSumTimestampResample" +"deltaSumTimestampSimpleState" +"deltaSumTimestampState" +"demangle" +"dense_rank" +"dense_rankArgMax" +"dense_rankArgMin" +"dense_rankArray" +"dense_rankDistinct" +"dense_rankForEach" +"dense_rankIf" +"dense_rankMap" +"dense_rankMerge" +"dense_rankNull" +"dense_rankOrDefault" +"dense_rankOrNull" +"dense_rankResample" +"dense_rankSimpleState" +"dense_rankState" +"detectCharset" +"detectLanguage" +"detectLanguageMixed" +"detectLanguageUnknown" +"detectProgrammingLanguage" +"detectTonality" +"dictGet" +"dictGetAll" +"dictGetChildren" +"dictGetDate" +"dictGetDateOrDefault" +"dictGetDateTime" +"dictGetDateTimeOrDefault" +"dictGetDescendants" +"dictGetFloat32" +"dictGetFloat32OrDefault" +"dictGetFloat64" +"dictGetFloat64OrDefault" +"dictGetHierarchy" +"dictGetIPv4" +"dictGetIPv4OrDefault" +"dictGetIPv6" +"dictGetIPv6OrDefault" +"dictGetInt16" +"dictGetInt16OrDefault" +"dictGetInt32" +"dictGetInt32OrDefault" +"dictGetInt64" +"dictGetInt64OrDefault" +"dictGetInt8" +"dictGetInt8OrDefault" +"dictGetOrDefault" +"dictGetOrNull" +"dictGetString" +"dictGetStringOrDefault" +"dictGetUInt16" +"dictGetUInt16OrDefault" +"dictGetUInt32" +"dictGetUInt32OrDefault" +"dictGetUInt64" +"dictGetUInt64OrDefault" +"dictGetUInt8" +"dictGetUInt8OrDefault" +"dictGetUUID" +"dictGetUUIDOrDefault" +"dictHas" +"dictIsIn" +"displayName" +"distanceL1" +"distanceL2" +"distanceL2Squared" +"distanceLinf" +"distanceLp" +"divide" +"divideDecimal" +"domain" +"domainRFC" +"domainWithoutWWW" +"domainWithoutWWWRFC" +"dotProduct" +"dumpColumnStructure" +"e" +"editDistance" +"empty" +"emptyArrayDate" +"emptyArrayDateTime" +"emptyArrayFloat32" +"emptyArrayFloat64" +"emptyArrayInt16" +"emptyArrayInt32" +"emptyArrayInt64" +"emptyArrayInt8" +"emptyArrayString" +"emptyArrayToSingle" +"emptyArrayUInt16" +"emptyArrayUInt32" +"emptyArrayUInt64" +"emptyArrayUInt8" +"enabledProfiles" +"enabledRoles" +"encodeURLComponent" +"encodeURLFormComponent" +"encodeXMLComponent" +"encrypt" +"endsWith" +"endsWithUTF8" +"entropy" +"entropyArgMax" +"entropyArgMin" +"entropyArray" +"entropyDistinct" +"entropyForEach" +"entropyIf" +"entropyMap" +"entropyMerge" +"entropyNull" +"entropyOrDefault" +"entropyOrNull" +"entropyResample" +"entropySimpleState" +"entropyState" +"equals" +"erf" +"erfc" +"errorCodeToName" +"evalMLMethod" +"exp" +"exp10" +"exp2" +"exponentialMovingAverage" +"exponentialMovingAverageArgMax" +"exponentialMovingAverageArgMin" +"exponentialMovingAverageArray" +"exponentialMovingAverageDistinct" +"exponentialMovingAverageForEach" +"exponentialMovingAverageIf" +"exponentialMovingAverageMap" +"exponentialMovingAverageMerge" +"exponentialMovingAverageNull" +"exponentialMovingAverageOrDefault" +"exponentialMovingAverageOrNull" +"exponentialMovingAverageResample" +"exponentialMovingAverageSimpleState" +"exponentialMovingAverageState" +"exponentialTimeDecayedAvg" +"exponentialTimeDecayedAvgArgMax" +"exponentialTimeDecayedAvgArgMin" +"exponentialTimeDecayedAvgArray" +"exponentialTimeDecayedAvgDistinct" +"exponentialTimeDecayedAvgForEach" +"exponentialTimeDecayedAvgIf" +"exponentialTimeDecayedAvgMap" +"exponentialTimeDecayedAvgMerge" +"exponentialTimeDecayedAvgNull" +"exponentialTimeDecayedAvgOrDefault" +"exponentialTimeDecayedAvgOrNull" +"exponentialTimeDecayedAvgResample" +"exponentialTimeDecayedAvgSimpleState" +"exponentialTimeDecayedAvgState" +"exponentialTimeDecayedCount" +"exponentialTimeDecayedCountArgMax" +"exponentialTimeDecayedCountArgMin" +"exponentialTimeDecayedCountArray" +"exponentialTimeDecayedCountDistinct" +"exponentialTimeDecayedCountForEach" +"exponentialTimeDecayedCountIf" +"exponentialTimeDecayedCountMap" +"exponentialTimeDecayedCountMerge" +"exponentialTimeDecayedCountNull" +"exponentialTimeDecayedCountOrDefault" +"exponentialTimeDecayedCountOrNull" +"exponentialTimeDecayedCountResample" +"exponentialTimeDecayedCountSimpleState" +"exponentialTimeDecayedCountState" +"exponentialTimeDecayedMax" +"exponentialTimeDecayedMaxArgMax" +"exponentialTimeDecayedMaxArgMin" +"exponentialTimeDecayedMaxArray" +"exponentialTimeDecayedMaxDistinct" +"exponentialTimeDecayedMaxForEach" +"exponentialTimeDecayedMaxIf" +"exponentialTimeDecayedMaxMap" +"exponentialTimeDecayedMaxMerge" +"exponentialTimeDecayedMaxNull" +"exponentialTimeDecayedMaxOrDefault" +"exponentialTimeDecayedMaxOrNull" +"exponentialTimeDecayedMaxResample" +"exponentialTimeDecayedMaxSimpleState" +"exponentialTimeDecayedMaxState" +"exponentialTimeDecayedSum" +"exponentialTimeDecayedSumArgMax" +"exponentialTimeDecayedSumArgMin" +"exponentialTimeDecayedSumArray" +"exponentialTimeDecayedSumDistinct" +"exponentialTimeDecayedSumForEach" +"exponentialTimeDecayedSumIf" +"exponentialTimeDecayedSumMap" +"exponentialTimeDecayedSumMerge" +"exponentialTimeDecayedSumNull" +"exponentialTimeDecayedSumOrDefault" +"exponentialTimeDecayedSumOrNull" +"exponentialTimeDecayedSumResample" +"exponentialTimeDecayedSumSimpleState" +"exponentialTimeDecayedSumState" +"extract" +"extractAll" +"extractAllGroups" "extractAllGroupsHorizontal" "extractAllGroupsVertical" -"multiFuzzyMatchAllIndices" -"multiFuzzyMatchAnyIndex" -"multiMatchAnyIndex" -"replaceRegexpAll" -"emptyArrayInt16" -"dictGetOrDefault" -"replaceOne" -"emptyArrayInt32" -"extract" -"readWKTPolygon" -"notILike" -"geohashDecode" -"toModifiedJulianDay" -"notLike" -"countSubstringsCaseInsensitive" -"IPv4StringToNum" -"positionUTF8" -"hasToken" -"dictGetDate" -"multiSearchFirstPosition" -"multiSearchFirstIndexCaseInsensitiveUTF8" -"atan" -"rowNumberInAllBlocks" -"multiSearchFirstIndexCaseInsensitive" -"multiSearchAnyUTF8" -"parseDateTime32BestEffortOrZero" -"blockNumber" -"cutURLParameter" -"multiSearchFirstPositionCaseInsensitive" -"positionCaseInsensitiveUTF8" -"positionCaseInsensitive" -"splitByRegexp" -"toInt256OrZero" -"pointInPolygon" -"splitByWhitespace" -"lemmatize" -"synonyms" -"tryBase64Decode" -"multiSearchAllPositions" -"FROM_BASE64" -"replaceRegexpOne" -"subBitmap" -"defaultValueOfArgumentType" -"base64Decode" -"regionToContinent" -"ilike" -"fromModifiedJulianDay" -"TO_BASE64" -"dictIsIn" -"base64Encode" -"decodeXMLComponent" -"countMatchesCaseInsensitive" -"normalizeQuery" -"erf" -"trimBoth" -"fromUnixTimestamp64Nano" -"lessOrEquals" -"subtractQuarters" -"ngramSearch" -"readWKTRing" -"trimRight" -"endsWith" -"ngramDistanceCaseInsensitive" -"connectionId" -"initialQueryID" -"startsWith" -"tan" -"substring" +"extractGroups" +"extractKeyValuePairs" +"extractKeyValuePairsWithEscaping" +"extractTextFromHTML" +"extractURLParameter" +"extractURLParameterNames" +"extractURLParameters" +"factorial" +"farmFingerprint64" +"farmHash64" +"file" +"filesystemAvailable" +"filesystemCapacity" +"filesystemUnreserved" +"finalizeAggregation" +"firstLine" +"firstSignificantSubdomain" +"firstSignificantSubdomainCustom" +"firstSignificantSubdomainCustomRFC" +"firstSignificantSubdomainRFC" +"first_value" +"first_valueArgMax" +"first_valueArgMin" +"first_valueArray" +"first_valueDistinct" +"first_valueForEach" +"first_valueIf" +"first_valueMap" +"first_valueMerge" +"first_valueNull" +"first_valueOrDefault" +"first_valueOrNull" +"first_valueResample" +"first_valueSimpleState" +"first_valueState" +"first_value_respect_nulls" +"first_value_respect_nullsArgMax" +"first_value_respect_nullsArgMin" +"first_value_respect_nullsArray" +"first_value_respect_nullsDistinct" +"first_value_respect_nullsForEach" +"first_value_respect_nullsIf" +"first_value_respect_nullsMap" +"first_value_respect_nullsMerge" +"first_value_respect_nullsNull" +"first_value_respect_nullsOrDefault" +"first_value_respect_nullsOrNull" +"first_value_respect_nullsResample" +"first_value_respect_nullsSimpleState" +"first_value_respect_nullsState" +"flameGraph" +"flameGraphArgMax" +"flameGraphArgMin" +"flameGraphArray" +"flameGraphDistinct" +"flameGraphForEach" +"flameGraphIf" +"flameGraphMap" +"flameGraphMerge" +"flameGraphNull" +"flameGraphOrDefault" +"flameGraphOrNull" +"flameGraphResample" +"flameGraphSimpleState" +"flameGraphState" +"flatten" +"flattenTuple" +"floor" +"format" +"formatDateTime" +"formatDateTimeInJodaSyntax" +"formatQuery" +"formatQueryOrNull" +"formatQuerySingleLine" +"formatQuerySingleLineOrNull" +"formatReadableDecimalSize" +"formatReadableQuantity" +"formatReadableSize" +"formatReadableTimeDelta" "formatRow" "formatRowNoNewline" -"dictGetHierarchy" -"notEmpty" -"format" -"murmurHash3_64" -"reverseUTF8" -"arrayEnumerateUniqRanked" -"CRC64" -"pow" -"toInt16" -"reverse" -"lower" -"toStartOfDay" -"caseWithoutExpr" -"CHAR_LENGTH" -"lengthUTF8" -"length" -"bitTestAny" -"floor" -"round" -"reinterpretAsFixedString" -"reinterpretAsString" -"h3ToChildren" -"emptyArrayToSingle" -"tgamma" -"reinterpretAsInt128" -"visitParamExtractRaw" -"reinterpretAsInt64" -"reinterpretAsUInt32" -"reinterpretAsUInt16" -"fuzzBits" -"randomFixedString" -"tupleHammingDistance" -"JSONExtractBool" -"ngramSearchUTF8" -"generateUUIDv4" -"multiSearchAllPositionsCaseInsensitive" -"randConstant" -"leftPad" -"convertCharset" -"extractURLParameter" -"zookeeperSessionUptime" -"serverUUID" -"firstSignificantSubdomain" -"rand64" -"JSON_EXISTS" -"isIPAddressInRange" -"wordShingleSimHashCaseInsensitive" -"parseDateTime32BestEffortOrNull" -"cutToFirstSignificantSubdomainWithWWW" -"roundToExp2" -"mapSubtract" -"partitionId" -"intExp2" -"byteSize" -"errorCodeToName" -"toDateTime64" -"yesterday" -"initializeAggregation" -"globalVariable" -"countDigits" -"toStartOfHour" -"toWeek" -"isDecimalOverflow" -"visitParamExtractString" -"isConstant" -"getSetting" -"shardNum" -"__getScalar" -"position" -"__bitBoolMaskOr" -"cosh" -"basename" -"evalMLMethod" -"filesystemUnreserved" -"filesystemCapacity" -"reinterpretAsDate" -"filesystemAvailable" -"joinGet" -"globalNullInIgnoreSet" -"globalNotInIgnoreSet" -"toRelativeDayNum" -"splitByChar" -"notInIgnoreSet" -"globalInIgnoreSet" -"notNullIn" -"ngramSearchCaseInsensitive" -"notIn" -"trunc" -"JSONExtractArrayRaw" -"version" -"joinGetOrNull" -"globalIn" -"toDate32" -"lowCardinalityIndices" -"toLowCardinality" -"runningConcurrency" -"runningDifferenceStartingWithFirstValue" -"bitCount" -"timezoneOf" -"timezone" -"uptime" -"pi" -"extractTextFromHTML" -"UUIDStringToNum" -"roundDown" -"IPv6NumToString" -"throwIf" -"ifNotFinite" -"FROM_UNIXTIME" -"isInfinite" -"arrayProduct" -"polygonsIntersectionSpherical" -"toRelativeHourNum" -"hasColumnInTable" -"bar" -"JSONExtractKeysAndValues" -"replicate" -"arrayJoin" -"s2RectUnion" -"ngramDistanceUTF8" -"identity" -"indexHint" -"ignore" -"regionHierarchy" -"toDateTime64OrNull" -"polygonsWithinCartesian" -"materialize" -"sleepEachRow" -"parseDateTimeBestEffortUSOrNull" -"neighbor" -"splitByString" -"rowNumberInBlock" -"blockSize" -"defaultValueOfTypeName" -"hiveHash" -"randomStringUTF8" -"blockSerializedSize" -"visibleWidth" -"fullHostName" -"hostName" -"arrayFilter" -"defaultRoles" -"currentRoles" -"toUInt64OrZero" -"dictGetUInt16" -"DATABASE" -"toUInt8OrZero" -"reinterpretAsInt8" -"not" -"log1p" -"and" -"wordShingleMinHash" -"arrayDifference" -"arrayCumSumNonNegative" -"wordShingleSimHash" -"arrayCumSum" -"arraySort" -"arrayPartialSort" -"arrayPartialReverseSort" -"toDaysSinceYearZero" +"fragment" "fromDaysSinceYearZero" "fromDaysSinceYearZero32" -"dumpColumnStructure" -"multiSearchFirstIndex" -"arrayReverseSplit" -"ifNull" -"normalizeQueryKeepNames" -"xxHash64" -"in" -"arrayReverseFill" -"erfc" -"cutToFirstSignificantSubdomainCustomWithWWW" -"arrayFill" -"substringUTF8" -"arrayFirst" -"arrayAvg" -"cos" -"arrayMax" -"toColumnTypeName" -"arrayMin" -"toIntervalDay" -"reinterpretAsFloat32" -"arrayAll" -"or" -"arrayExists" -"trimLeft" -"arrayMap" -"polygonsSymDifferenceCartesian" -"gccMurmurHash" -"exp2" -"polygonPerimeterCartesian" -"SHA384" -"murmurHash3_128" -"polygonConvexHullCartesian" -"murmurHash3_32" -"h3GetBaseCell" -"murmurHash2_64" -"globalNullIn" -"javaHashUTF16LE" -"URLHash" -"reinterpretAsInt16" -"regionToCountry" -"toUUID" -"intHash32" -"metroHash64" -"arrayPushFront" -"bitmapAndnotCardinality" -"toFloat32" -"sipHash64" -"JSONExtractFloat" -"SHA512" -"JSONExtractUInt" -"arrayReduce" -"SHA256" -"array" -"parseDateTimeBestEffortUSOrZero" -"toInt32OrNull" -"MD5" -"randomString" -"__bitBoolMaskAnd" -"hasSubstr" -"formatReadableQuantity" -"replaceAll" -"toIntervalWeek" -"toRelativeSecondNum" -"demangle" -"toNullable" -"concat" -"formatReadableDecimalSize" -"formatReadableSize" -"shardCount" +"fromModifiedJulianDay" "fromModifiedJulianDayOrNull" -"toInt128OrZero" -"bitmaskToList" -"xor" -"bitPositionsToArray" -"empty" -"ngramMinHashArg" -"_CAST" -"dictGetStringOrDefault" -"arrayZip" -"dictGetUUIDOrDefault" -"dictGetFloat64OrDefault" -"dictGetInt64OrDefault" -"today" -"dictGetUInt32OrDefault" -"countSubstringsCaseInsensitiveUTF8" -"h3IsPentagon" -"dictGetUInt16OrDefault" -"dictGetDescendants" -"dictGetString" -"inIgnoreSet" -"dictGetUUID" -"arraySlice" -"reinterpretAsInt32" -"dictGetFloat64" -"globalNotNullInIgnoreSet" -"arrayUniq" -"polygonAreaCartesian" -"intDiv" -"cityHash64" -"regionToTopContinent" -"toInt32" -"e" -"wordShingleSimHashCaseInsensitiveUTF8" -"JSON_QUERY" -"isIPv4String" -"UUIDNumToString" -"modulo" -"reinterpret" -"unbin" -"regionToDistrict" -"sinh" -"date_trunc" -"__bitSwapLastTwo" -"formatDateTime" -"dateDiff" -"tanh" -"multiSearchAnyCaseInsensitiveUTF8" -"upperUTF8" -"bitmaskToArray" -"arrayAUC" -"multiMatchAllIndices" -"hasThreadFuzzer" -"toDayOfWeek" -"isFinite" -"transform" -"timezoneOffset" -"toTimezone" -"cutQueryString" -"subtractYears" -"tcpPort" -"subtractHours" -"tid" -"toDateTime64OrZero" -"subtractSeconds" -"alphaTokens" -"negate" -"file" -"roundAge" -"MACStringToOUI" -"addQuarters" -"arrayResize" -"addWeeks" -"HOUR" -"addHours" -"visitParamExtractUInt" -"caseWithExpression" -"rand" -"addSeconds" -"splitByNonAlpha" -"arrayStringConcat" -"randomPrintableASCII" -"h3HexAreaM2" -"toYYYYMMDDhhmmss" -"toInt16OrNull" -"timeSlots" -"timeSlot" -"dictGetInt16" -"queryString" -"javaHash" -"acosh" -"toRelativeMinuteNum" -"toRelativeYearNum" -"unhex" -"toInt8OrNull" -"regionToPopulation" -"toStartOfISOYear" -"getSizeOfEnumType" -"toStartOfInterval" -"toYYYYMMDD" -"multiSearchFirstPositionUTF8" -"CRC32" -"toStartOfFifteenMinutes" -"emptyArrayUInt8" -"dictGetUInt8" -"toStartOfFiveMinutes" -"cbrt" -"toStartOfMinute" -"dictGet" -"toStartOfSecond" -"hasTokenCaseInsensitive" -"toStartOfYear" -"farmHash64" -"toStartOfMonth" -"queryID" -"wordShingleMinHashCaseInsensitive" -"dictGetDateOrDefault" -"dictGetUInt64OrDefault" -"arrayElement" -"sleep" -"dictHas" -"enabledProfiles" -"toDecimal32OrNull" -"multiFuzzyMatchAny" -"toStartOfWeek" -"isValidUTF8" -"dictGetInt8" -"dictGetInt32OrDefault" -"toMonday" -"multiSearchFirstIndexUTF8" -"lowCardinalityKeys" -"halfMD5" -"domainWithoutWWW" -"MINUTE" -"addDays" -"toMinute" -"JSONExtractRaw" -"toUInt16OrNull" -"bin" -"toDayOfYear" -"log" -"DAYOFMONTH" -"toUInt32" -"toDecimal32" -"s2RectAdd" -"dictGetFloat32OrDefault" -"xxHash32" -"DAY" -"toISOWeek" -"ngramMinHash" -"toMonth" -"countMatches" -"toQuarter" -"toISOYear" -"YEAR" -"toIntervalYear" -"hypot" -"multiSearchAny" -"toIntervalQuarter" -"toSecond" -"accurate_CastOrNull" -"reinterpretAsUInt64" -"defaultProfiles" -"toIntervalMonth" -"bitRotateRight" -"arrayReverseSort" -"toInt8OrZero" -"SHA1" -"multiSearchFirstPositionCaseInsensitiveUTF8" -"cutFragment" -"toDecimal256" -"toIntervalSecond" -"toUInt256OrZero" -"toStringCutToZero" -"normalizedQueryHashKeepNames" -"coalesce" -"parseDateTime64BestEffortOrNull" -"toIntervalHour" -"bitmapSubsetLimit" -"parseDateTimeBestEffortOrNull" -"match" -"fragment" -"repeat" -"parseDateTimeBestEffortOrZero" -"toDecimal128OrNull" -"parseDateTimeBestEffort" -"simpleJSONExtractUInt" -"multiMatchAny" -"toDecimal256OrNull" -"roundDuration" -"addMinutes" -"toStartOfQuarter" -"exp10" -"regionIn" -"toDecimal64OrNull" -"toDateTimeOrNull" -"simpleJSONExtractFloat" -"reinterpretAsUUID" -"dictGetDateTimeOrDefault" -"greater" -"toRelativeMonthNum" -"toDate32OrNull" -"h3IndexesAreNeighbors" -"toFloat32OrNull" -"toInt256OrNull" -"leftPadUTF8" -"regionToArea" -"toInt64OrNull" -"toModifiedJulianDayOrNull" -"toUInt64OrNull" -"polygonsEqualsCartesian" -"accurateCast" -"toUInt32OrNull" -"encodeXMLComponent" -"toUInt8OrNull" -"toUUIDOrZero" -"arrayEnumerateDenseRanked" -"toYYYYMM" -"toDecimal256OrZero" -"fromUnixTimestamp64Milli" -"CHARACTER_LENGTH" -"isNull" -"isNaN" -"multiSearchAllPositionsCaseInsensitiveUTF8" -"FQDN" -"currentProfiles" -"toDateTime32" -"arraySplit" -"toIPv6" -"toDecimal32OrZero" -"bitmapCardinality" -"arrayFlatten" -"dateName" -"s2RectIntersection" -"CAST" -"caseWithExpr" -"nullIf" -"toDateTimeOrZero" -"bitOr" -"CRC32IEEE" -"emptyArrayDate" -"toUInt128OrZero" -"toUInt128OrNull" -"toFloat32OrZero" -"SECOND" -"arrayCompact" -"bitmapMin" -"toInt16OrZero" -"reinterpretAsUInt256" -"dictGetOrNull" -"map" -"bitmapHasAny" -"emptyArrayUInt16" -"cutIPv6" -"toUInt32OrZero" -"toTypeName" -"polygonsUnionCartesian" -"char" -"mapContains" -"emptyArrayDateTime" -"toUInt16OrZero" -"toInt64" -"wkt" -"dictGetUInt64" -"JSONHas" -"runningDifference" -"getServerPort" -"IPv4ToIPv6" -"__bitWrapperFunc" -"multiply" -"accurate_Cast" -"toDecimal64" -"toString" -"arrayIntersect" -"dictGetUInt8OrDefault" -"toTime" -"hasAll" -"now" -"parseDateTime64BestEffortOrZero" -"rightPadUTF8" -"emptyArrayInt64" -"simpleJSONExtractInt" -"toDecimal128" -"toRelativeWeekNum" -"toDateOrNull" -"topLevelDomain" -"greatest" -"isIPv6String" -"toHour" -"toFixedString" -"bitmapOrCardinality" -"toInt128" -"toDateOrZero" -"emptyArrayUInt64" -"toUInt128" -"isNotNull" -"toUInt16" -"MACNumToString" -"multiSearchAllPositionsUTF8" -"toUInt8" -"regionToCity" -"DAYOFYEAR" -"simpleJSONExtractRaw" -"appendTrailingCharIfAbsent" -"toUInt64" -"regionToName" -"IPv4NumToStringClassC" -"upper" -"greaterOrEquals" -"s2RectContains" -"toDate" -"regexpQuoteMeta" -"readWKTMultiPolygon" -"emptyArrayString" -"bitmapOr" -"cutWWW" -"emptyArrayInt8" -"less" -"readWKTPoint" -"reinterpretAsDateTime" -"notEquals" -"geoToS2" -"nullInIgnoreSet" -"globalNotNullIn" -"arrayConcat" -"mapPopulateSeries" -"parseDateTimeBestEffortUS" -"toDayOfMonth" -"gcd" -"protocol" -"currentUser" -"if" -"caseWithoutExpression" -"currentDatabase" -"log10" -"moduloOrZero" -"MD4" -"toInt64OrZero" -"arrayWithConstant" -"IPv4NumToString" -"dictGetUInt32" -"multiSearchAnyCaseInsensitive" -"countSubstrings" -"reinterpretAsFloat64" -"plus" -"dictGetChildren" -"IPv6CIDRToRange" -"toIntervalMinute" -"bitRotateLeft" -"geoDistance" -"mapValues" -"toIPv4" -"extractGroups" -"toYearWeek" -"polygonsDistanceCartesian" -"lowerUTF8" -"toUInt256OrNull" -"IPv4CIDRToRange" -"MACStringToNum" -"toFloat64OrNull" -"sipHash128" -"toFloat64" -"arrayPopFront" -"bitmapAnd" -"bitmapHasAll" -"subtractWeeks" -"arrayFirstIndex" -"bitmapXorCardinality" -"bitmapAndnot" -"asinh" -"dictGetInt32" -"toFloat64OrZero" -"toInt8" -"polygonsWithinSpherical" -"bitmapXor" -"intExp10" -"dictGetDateTime" -"h3ToGeoBoundary" -"decodeURLComponent" -"bitAnd" -"toUUIDOrNull" -"notNullInIgnoreSet" -"tupleElement" -"tupleToNameValuePairs" -"tuple" -"ceil" -"DAYOFWEEK" -"toDateTime" -"bitNot" -"rightPad" -"has" -"bitmapToArray" -"arrayCount" -"arrayDistinct" -"greatCircleAngle" -"IPv6StringToNum" -"dictGetInt64" -"toUnixTimestamp" -"getMacro" -"runningAccumulate" -"multiIf" -"abs" -"toValidUTF8" -"arrayEnumerateUniq" -"moduloLegacy" -"JSONExtractInt" -"subtractMonths" -"dateTimeToSnowflake" -"cutToFirstSignificantSubdomain" -"roundBankers" -"toUInt256" -"arrayEnumerateDense" -"stem" -"bitmapAndCardinality" -"bitTestAll" -"normalizedQueryHash" -"toYear" -"emptyArrayFloat64" -"bitmapTransform" -"JSONExtract" -"bitShiftRight" -"bitmapContains" -"sqrt" -"minus" -"QUARTER" -"parseDateTime64BestEffort" -"h3IsValid" -"h3EdgeAngle" -"concatAssumeInjective" -"emptyArrayUInt32" -"intHash64" -"enabledRoles" -"fromUnixTimestamp64Micro" -"range" -"subtractMinutes" -"farmFingerprint64" -"arrayReduceInRanges" -"bitmapBuild" -"bitmapSubsetInRange" -"arrayPopBack" -"h3ToGeo" -"hex" -"arrayPushBack" -"subtractDays" -"buildId" -"URLPathHierarchy" -"addMonths" -"parseDateTime32BestEffort" -"indexOf" -"polygonsUnionSpherical" -"arraySum" -"dictGetFloat32" -"ngramSimHashCaseInsensitiveUTF8" -"finalizeAggregation" -"divide" -"netloc" -"bitShiftLeft" -"toInt32OrZero" -"extractAll" -"toInt128OrNull" -"bitXor" -"lcm" -"accurateCastOrNull" -"toDate32OrZero" -"simpleJSONExtractString" -"least" -"nullIn" -"MONTH" -"arrayReverse" -"arrayShuffle" -"arrayPartialShuffle" -"now64" -"DATE" -"addressToLine" -"globalNotIn" -"dateTime64ToSnowflake" -"dictGetInt8OrDefault" -"exp" -"toRelativeQuarterNum" -"arrayEnumerate" -"mapAdd" -"murmurHash2_32" -"toStartOfTenMinutes" -"intDivOrZero" -"addYears" -"queryStringAndFragment" -"emptyArrayFloat32" -"countEqual" -"pointInEllipses" -"like" -"toDecimal64OrZero" -"reinterpretAsInt256" -"hasAny" -"ln" -"replace" -"rpad" -"mid" -"lcase" +"fromUTCTimestamp" "fromUnixTimestamp" -"truncate" -"ceiling" -"ucase" -"rand32" -"power" -"initial_query_id" -"timeZone" -"hostname" -"yearweek" -"timeZoneOffset" -"extractAllGroups" -"lpad" -"substr" -"INET6_NTOA" -"INET6_ATON" -"INET_ATON" -"timeZoneOf" -"locate" -"INET_NTOA" -"connection_id" -"user" -"toTimeZone" -"week" -"flatten" -"query_id" -"dateTrunc" -"mod" -"lagInFrame" -"dense_rank" -"rank" -"exponentialMovingAverage" -"sparkbar" -"singleValueOrNull" -"studentTTest" -"aggThrow" -"categoricalInformationValue" -"groupArrayMovingAvg" -"groupArrayMovingSum" -"simpleLinearRegression" -"entropy" -"quantilesBFloat16" -"maxIntersectionsPosition" -"groupBitmapXor" -"groupBitmap" -"skewPop" -"groupBitXor" -"groupBitOr" -"groupBitmapAnd" -"topKWeighted" -"stochasticLinearRegression" -"corr" -"corrMatrix" -"uniqCombined64" -"intervalLengthSum" -"uniqCombined" -"rankCorr" -"maxMap" -"minMap" -"sumMapFiltered" -"quantileExactWeighted" -"sumMapFilteredWithOverflow" -"sumMap" -"histogram" -"quantiles" -"sum" -"covarPop" -"covarPopMatrix" -"row_number" -"kurtPop" -"kurtSamp" -"skewSamp" -"uniqExact" -"sumMapWithOverflow" -"stddevSamp" -"varPop" -"corrStable" -"quantileTimingWeighted" -"covarPopStable" -"stddevSampStable" -"varSamp" -"topK" -"last_value" -"mannWhitneyUTest" -"maxIntersections" -"quantilesExact" -"uniqHLL12" -"quantileBFloat16" -"uniq" -"min" -"sequenceNextNode" -"quantilesTimingWeighted" -"boundingRatio" -"any" -"anyLast" -"deltaSum" -"retention" -"sequenceMatch" -"uniqUpTo" -"windowFunnel" -"deltaSumTimestamp" -"varSampStable" -"uniqTheta" -"quantilesExactWeighted" -"max" -"quantilesBFloat16Weighted" -"quantileBFloat16Weighted" -"sumKahan" -"quantilesTDigestWeighted" -"groupBitAnd" -"quantileTDigest" -"quantileTDigestWeighted" -"argMax" -"quantileDeterministic" -"quantilesTDigest" -"stochasticLogisticRegression" -"argMin" -"avg" -"covarSampStable" -"quantilesTiming" -"welchTTest" -"covarSamp" -"covarSampMatrix" -"varPopStable" -"quantileTiming" -"quantileExactInclusive" -"quantileExactHigh" -"groupArraySample" -"quantilesExactLow" -"groupBitmapOr" -"first_value" -"quantileExactExclusive" -"quantileExact" -"sumCount" -"groupArrayInsertAt" -"quantilesExactHigh" -"sumWithOverflow" -"sequenceCount" -"quantilesDeterministic" -"groupUniqArray" +"fromUnixTimestamp64Micro" +"fromUnixTimestamp64Milli" +"fromUnixTimestamp64Nano" +"fromUnixTimestampInJodaSyntax" +"from_utc_timestamp" +"fullHostName" +"fuzzBits" +"gccMurmurHash" +"gcd" +"generateRandomStructure" +"generateULID" +"generateUUIDv4" +"geoDistance" +"geoToH3" +"geoToS2" +"geohashDecode" +"geohashEncode" +"geohashesInBox" +"getMacro" +"getOSKernelVersion" +"getServerPort" +"getSetting" +"getSizeOfEnumType" +"getSubcolumn" +"getTypeSerializationStreams" +"globalIn" +"globalInIgnoreSet" +"globalNotIn" +"globalNotInIgnoreSet" +"globalNotNullIn" +"globalNotNullInIgnoreSet" +"globalNullIn" +"globalNullInIgnoreSet" +"globalVariable" +"greatCircleAngle" +"greatCircleDistance" +"greater" +"greaterOrEquals" +"greatest" "groupArray" -"anyHeavy" -"stddevPop" -"quantile" -"leadInFrame" -"quantilesExactExclusive" -"count" -"quantilesExactInclusive" -"stddevPopStable" -"quantileExactLow" -"avgWeighted" -"BIT_AND" -"VAR_SAMP" -"COVAR_SAMP" -"VAR_POP" -"medianTDigest" -"medianBFloat16" -"medianTimingWeighted" -"medianTiming" -"medianExactHigh" -"BIT_OR" -"medianDeterministic" -"STDDEV_POP" -"STDDEV_SAMP" -"medianExactLow" -"medianTDigestWeighted" -"medianExact" -"COVAR_POP" -"medianBFloat16Weighted" -"medianExactWeighted" -"BIT_XOR" -"median" -"log" -"replaceAll" -"rightPad" -"substring" -"lower" -"FROM_UNIXTIME" -"trunc" -"ceil" -"upper" -"rand" -"pow" -"initialQueryID" -"timezone" -"hostName" -"toYearWeek" -"timezoneOffset" -"extractAllGroupsVertical" -"leftPad" -"substring" -"IPv6NumToString" -"IPv6StringToNum" -"IPv4StringToNum" -"timezoneOf" -"position" -"IPv4NumToString" -"connectionid" -"currentUser" -"toTimezone" -"toWeek" -"arrayFlatten" -"queryID" -"date_trunc" -"modulo" +"groupArrayArgMax" +"groupArrayArgMin" +"groupArrayArray" +"groupArrayDistinct" +"groupArrayForEach" +"groupArrayIf" +"groupArrayInsertAt" +"groupArrayInsertAtArgMax" +"groupArrayInsertAtArgMin" +"groupArrayInsertAtArray" +"groupArrayInsertAtDistinct" +"groupArrayInsertAtForEach" +"groupArrayInsertAtIf" +"groupArrayInsertAtMap" +"groupArrayInsertAtMerge" +"groupArrayInsertAtNull" +"groupArrayInsertAtOrDefault" +"groupArrayInsertAtOrNull" +"groupArrayInsertAtResample" +"groupArrayInsertAtSimpleState" +"groupArrayInsertAtState" +"groupArrayIntersect" +"groupArrayIntersectArgMax" +"groupArrayIntersectArgMin" +"groupArrayIntersectArray" +"groupArrayIntersectDistinct" +"groupArrayIntersectForEach" +"groupArrayIntersectIf" +"groupArrayIntersectMap" +"groupArrayIntersectMerge" +"groupArrayIntersectNull" +"groupArrayIntersectOrDefault" +"groupArrayIntersectOrNull" +"groupArrayIntersectResample" +"groupArrayIntersectSimpleState" +"groupArrayIntersectState" +"groupArrayLast" +"groupArrayLastArgMax" +"groupArrayLastArgMin" +"groupArrayLastArray" +"groupArrayLastDistinct" +"groupArrayLastForEach" +"groupArrayLastIf" +"groupArrayLastMap" +"groupArrayLastMerge" +"groupArrayLastNull" +"groupArrayLastOrDefault" +"groupArrayLastOrNull" +"groupArrayLastResample" +"groupArrayLastSimpleState" +"groupArrayLastState" +"groupArrayMap" +"groupArrayMerge" +"groupArrayMovingAvg" +"groupArrayMovingAvgArgMax" +"groupArrayMovingAvgArgMin" +"groupArrayMovingAvgArray" +"groupArrayMovingAvgDistinct" +"groupArrayMovingAvgForEach" +"groupArrayMovingAvgIf" +"groupArrayMovingAvgMap" +"groupArrayMovingAvgMerge" +"groupArrayMovingAvgNull" +"groupArrayMovingAvgOrDefault" +"groupArrayMovingAvgOrNull" +"groupArrayMovingAvgResample" +"groupArrayMovingAvgSimpleState" +"groupArrayMovingAvgState" +"groupArrayMovingSum" +"groupArrayMovingSumArgMax" +"groupArrayMovingSumArgMin" +"groupArrayMovingSumArray" +"groupArrayMovingSumDistinct" +"groupArrayMovingSumForEach" +"groupArrayMovingSumIf" +"groupArrayMovingSumMap" +"groupArrayMovingSumMerge" +"groupArrayMovingSumNull" +"groupArrayMovingSumOrDefault" +"groupArrayMovingSumOrNull" +"groupArrayMovingSumResample" +"groupArrayMovingSumSimpleState" +"groupArrayMovingSumState" +"groupArrayNull" +"groupArrayOrDefault" +"groupArrayOrNull" +"groupArrayResample" +"groupArraySample" +"groupArraySampleArgMax" +"groupArraySampleArgMin" +"groupArraySampleArray" +"groupArraySampleDistinct" +"groupArraySampleForEach" +"groupArraySampleIf" +"groupArraySampleMap" +"groupArraySampleMerge" +"groupArraySampleNull" +"groupArraySampleOrDefault" +"groupArraySampleOrNull" +"groupArraySampleResample" +"groupArraySampleSimpleState" +"groupArraySampleState" +"groupArraySimpleState" +"groupArraySorted" +"groupArraySortedArgMax" +"groupArraySortedArgMin" +"groupArraySortedArray" +"groupArraySortedDistinct" +"groupArraySortedForEach" +"groupArraySortedIf" +"groupArraySortedMap" +"groupArraySortedMerge" +"groupArraySortedNull" +"groupArraySortedOrDefault" +"groupArraySortedOrNull" +"groupArraySortedResample" +"groupArraySortedSimpleState" +"groupArraySortedState" +"groupArrayState" "groupBitAnd" -"varSamp" -"covarSamp" -"varPop" -"quantileTDigest" -"quantileBFloat16" -"quantileTimingWeighted" -"quantileTiming" -"quantileExactHigh" +"groupBitAndArgMax" +"groupBitAndArgMin" +"groupBitAndArray" +"groupBitAndDistinct" +"groupBitAndForEach" +"groupBitAndIf" +"groupBitAndMap" +"groupBitAndMerge" +"groupBitAndNull" +"groupBitAndOrDefault" +"groupBitAndOrNull" +"groupBitAndResample" +"groupBitAndSimpleState" +"groupBitAndState" "groupBitOr" -"quantileDeterministic" -"stddevPop" -"stddevSamp" -"quantileExactLow" -"quantileTDigestWeighted" -"quantileExact" -"covarPop" -"quantileBFloat16Weighted" -"quantileExactWeighted" +"groupBitOrArgMax" +"groupBitOrArgMin" +"groupBitOrArray" +"groupBitOrDistinct" +"groupBitOrForEach" +"groupBitOrIf" +"groupBitOrMap" +"groupBitOrMerge" +"groupBitOrNull" +"groupBitOrOrDefault" +"groupBitOrOrNull" +"groupBitOrResample" +"groupBitOrSimpleState" +"groupBitOrState" "groupBitXor" +"groupBitXorArgMax" +"groupBitXorArgMin" +"groupBitXorArray" +"groupBitXorDistinct" +"groupBitXorForEach" +"groupBitXorIf" +"groupBitXorMap" +"groupBitXorMerge" +"groupBitXorNull" +"groupBitXorOrDefault" +"groupBitXorOrNull" +"groupBitXorResample" +"groupBitXorSimpleState" +"groupBitXorState" +"groupBitmap" +"groupBitmapAnd" +"groupBitmapAndArgMax" +"groupBitmapAndArgMin" +"groupBitmapAndArray" +"groupBitmapAndDistinct" +"groupBitmapAndForEach" +"groupBitmapAndIf" +"groupBitmapAndMap" +"groupBitmapAndMerge" +"groupBitmapAndNull" +"groupBitmapAndOrDefault" +"groupBitmapAndOrNull" +"groupBitmapAndResample" +"groupBitmapAndSimpleState" +"groupBitmapAndState" +"groupBitmapArgMax" +"groupBitmapArgMin" +"groupBitmapArray" +"groupBitmapDistinct" +"groupBitmapForEach" +"groupBitmapIf" +"groupBitmapMap" +"groupBitmapMerge" +"groupBitmapNull" +"groupBitmapOr" +"groupBitmapOrArgMax" +"groupBitmapOrArgMin" +"groupBitmapOrArray" +"groupBitmapOrDefault" +"groupBitmapOrDistinct" +"groupBitmapOrForEach" +"groupBitmapOrIf" +"groupBitmapOrMap" +"groupBitmapOrMerge" +"groupBitmapOrNull" +"groupBitmapOrNull" +"groupBitmapOrOrDefault" +"groupBitmapOrOrNull" +"groupBitmapOrResample" +"groupBitmapOrSimpleState" +"groupBitmapOrState" +"groupBitmapResample" +"groupBitmapSimpleState" +"groupBitmapState" +"groupBitmapXor" +"groupBitmapXorArgMax" +"groupBitmapXorArgMin" +"groupBitmapXorArray" +"groupBitmapXorDistinct" +"groupBitmapXorForEach" +"groupBitmapXorIf" +"groupBitmapXorMap" +"groupBitmapXorMerge" +"groupBitmapXorNull" +"groupBitmapXorOrDefault" +"groupBitmapXorOrNull" +"groupBitmapXorResample" +"groupBitmapXorSimpleState" +"groupBitmapXorState" +"groupUniqArray" +"groupUniqArrayArgMax" +"groupUniqArrayArgMin" +"groupUniqArrayArray" +"groupUniqArrayDistinct" +"groupUniqArrayForEach" +"groupUniqArrayIf" +"groupUniqArrayMap" +"groupUniqArrayMerge" +"groupUniqArrayNull" +"groupUniqArrayOrDefault" +"groupUniqArrayOrNull" +"groupUniqArrayResample" +"groupUniqArraySimpleState" +"groupUniqArrayState" +"h3CellAreaM2" +"h3CellAreaRads2" +"h3Distance" +"h3EdgeAngle" +"h3EdgeLengthKm" +"h3EdgeLengthM" +"h3ExactEdgeLengthKm" +"h3ExactEdgeLengthM" +"h3ExactEdgeLengthRads" +"h3GetBaseCell" +"h3GetDestinationIndexFromUnidirectionalEdge" +"h3GetFaces" +"h3GetIndexesFromUnidirectionalEdge" +"h3GetOriginIndexFromUnidirectionalEdge" +"h3GetPentagonIndexes" +"h3GetRes0Indexes" +"h3GetResolution" +"h3GetUnidirectionalEdge" +"h3GetUnidirectionalEdgeBoundary" +"h3GetUnidirectionalEdgesFromHexagon" +"h3HexAreaKm2" +"h3HexAreaM2" +"h3HexRing" +"h3IndexesAreNeighbors" +"h3IsPentagon" +"h3IsResClassIII" +"h3IsValid" +"h3Line" +"h3NumHexagons" +"h3PointDistKm" +"h3PointDistM" +"h3PointDistRads" +"h3ToCenterChild" +"h3ToChildren" +"h3ToGeo" +"h3ToGeoBoundary" +"h3ToParent" +"h3ToString" +"h3UnidirectionalEdgeIsValid" +"h3kRing" +"halfMD5" +"has" +"hasAll" +"hasAny" +"hasColumnInTable" +"hasSubsequence" +"hasSubsequenceCaseInsensitive" +"hasSubsequenceCaseInsensitiveUTF8" +"hasSubsequenceUTF8" +"hasSubstr" +"hasThreadFuzzer" +"hasToken" +"hasTokenCaseInsensitive" +"hasTokenCaseInsensitiveOrNull" +"hasTokenOrNull" +"hex" +"histogram" +"histogramArgMax" +"histogramArgMin" +"histogramArray" +"histogramDistinct" +"histogramForEach" +"histogramIf" +"histogramMap" +"histogramMerge" +"histogramNull" +"histogramOrDefault" +"histogramOrNull" +"histogramResample" +"histogramSimpleState" +"histogramState" +"hiveHash" +"hop" +"hopEnd" +"hopStart" +"hostName" +"hostname" +"hypot" +"identity" +"idnaDecode" +"idnaEncode" +"if" +"ifNotFinite" +"ifNull" +"ignore" +"ilike" +"in" +"inIgnoreSet" +"indexHint" +"indexOf" +"initcap" +"initcapUTF8" +"initialQueryID" +"initial_query_id" +"initializeAggregation" +"instr" +"intDiv" +"intDivOrZero" +"intExp10" +"intExp2" +"intHash32" +"intHash64" +"intervalLengthSum" +"intervalLengthSumArgMax" +"intervalLengthSumArgMin" +"intervalLengthSumArray" +"intervalLengthSumDistinct" +"intervalLengthSumForEach" +"intervalLengthSumIf" +"intervalLengthSumMap" +"intervalLengthSumMerge" +"intervalLengthSumNull" +"intervalLengthSumOrDefault" +"intervalLengthSumOrNull" +"intervalLengthSumResample" +"intervalLengthSumSimpleState" +"intervalLengthSumState" +"isConstant" +"isDecimalOverflow" +"isFinite" +"isIPAddressInRange" +"isIPv4String" +"isIPv6String" +"isInfinite" +"isNaN" +"isNotDistinctFrom" +"isNotNull" +"isNull" +"isNullable" +"isValidJSON" +"isValidUTF8" +"isZeroOrNull" +"jaroSimilarity" +"jaroWinklerSimilarity" +"javaHash" +"javaHashUTF16LE" +"joinGet" +"joinGetOrNull" +"jsonMergePatch" +"jumpConsistentHash" +"kafkaMurmurHash" +"kolmogorovSmirnovTest" +"kolmogorovSmirnovTestArgMax" +"kolmogorovSmirnovTestArgMin" +"kolmogorovSmirnovTestArray" +"kolmogorovSmirnovTestDistinct" +"kolmogorovSmirnovTestForEach" +"kolmogorovSmirnovTestIf" +"kolmogorovSmirnovTestMap" +"kolmogorovSmirnovTestMerge" +"kolmogorovSmirnovTestNull" +"kolmogorovSmirnovTestOrDefault" +"kolmogorovSmirnovTestOrNull" +"kolmogorovSmirnovTestResample" +"kolmogorovSmirnovTestSimpleState" +"kolmogorovSmirnovTestState" +"kostikConsistentHash" +"kql_array_sort_asc" +"kql_array_sort_desc" +"kurtPop" +"kurtPopArgMax" +"kurtPopArgMin" +"kurtPopArray" +"kurtPopDistinct" +"kurtPopForEach" +"kurtPopIf" +"kurtPopMap" +"kurtPopMerge" +"kurtPopNull" +"kurtPopOrDefault" +"kurtPopOrNull" +"kurtPopResample" +"kurtPopSimpleState" +"kurtPopState" +"kurtSamp" +"kurtSampArgMax" +"kurtSampArgMin" +"kurtSampArray" +"kurtSampDistinct" +"kurtSampForEach" +"kurtSampIf" +"kurtSampMap" +"kurtSampMerge" +"kurtSampNull" +"kurtSampOrDefault" +"kurtSampOrNull" +"kurtSampResample" +"kurtSampSimpleState" +"kurtSampState" +"lagInFrame" +"lagInFrameArgMax" +"lagInFrameArgMin" +"lagInFrameArray" +"lagInFrameDistinct" +"lagInFrameForEach" +"lagInFrameIf" +"lagInFrameMap" +"lagInFrameMerge" +"lagInFrameNull" +"lagInFrameOrDefault" +"lagInFrameOrNull" +"lagInFrameResample" +"lagInFrameSimpleState" +"lagInFrameState" +"largestTriangleThreeBuckets" +"largestTriangleThreeBucketsArgMax" +"largestTriangleThreeBucketsArgMin" +"largestTriangleThreeBucketsArray" +"largestTriangleThreeBucketsDistinct" +"largestTriangleThreeBucketsForEach" +"largestTriangleThreeBucketsIf" +"largestTriangleThreeBucketsMap" +"largestTriangleThreeBucketsMerge" +"largestTriangleThreeBucketsNull" +"largestTriangleThreeBucketsOrDefault" +"largestTriangleThreeBucketsOrNull" +"largestTriangleThreeBucketsResample" +"largestTriangleThreeBucketsSimpleState" +"largestTriangleThreeBucketsState" +"last_value" +"last_valueArgMax" +"last_valueArgMin" +"last_valueArray" +"last_valueDistinct" +"last_valueForEach" +"last_valueIf" +"last_valueMap" +"last_valueMerge" +"last_valueNull" +"last_valueOrDefault" +"last_valueOrNull" +"last_valueResample" +"last_valueSimpleState" +"last_valueState" +"last_value_respect_nulls" +"last_value_respect_nullsArgMax" +"last_value_respect_nullsArgMin" +"last_value_respect_nullsArray" +"last_value_respect_nullsDistinct" +"last_value_respect_nullsForEach" +"last_value_respect_nullsIf" +"last_value_respect_nullsMap" +"last_value_respect_nullsMerge" +"last_value_respect_nullsNull" +"last_value_respect_nullsOrDefault" +"last_value_respect_nullsOrNull" +"last_value_respect_nullsResample" +"last_value_respect_nullsSimpleState" +"last_value_respect_nullsState" +"lcase" +"lcm" +"leadInFrame" +"leadInFrameArgMax" +"leadInFrameArgMin" +"leadInFrameArray" +"leadInFrameDistinct" +"leadInFrameForEach" +"leadInFrameIf" +"leadInFrameMap" +"leadInFrameMerge" +"leadInFrameNull" +"leadInFrameOrDefault" +"leadInFrameOrNull" +"leadInFrameResample" +"leadInFrameSimpleState" +"leadInFrameState" +"least" +"left" +"leftPad" +"leftPadUTF8" +"leftUTF8" +"lemmatize" +"length" +"lengthUTF8" +"less" +"lessOrEquals" +"levenshteinDistance" +"lgamma" +"like" +"ln" +"locate" +"log" +"log10" +"log1p" +"log2" +"logTrace" +"lowCardinalityIndices" +"lowCardinalityKeys" +"lower" +"lowerUTF8" +"lpad" +"ltrim" +"lttb" +"lttbArgMax" +"lttbArgMin" +"lttbArray" +"lttbDistinct" +"lttbForEach" +"lttbIf" +"lttbMap" +"lttbMerge" +"lttbNull" +"lttbOrDefault" +"lttbOrNull" +"lttbResample" +"lttbSimpleState" +"lttbState" +"makeDate" +"makeDate32" +"makeDateTime" +"makeDateTime64" +"mannWhitneyUTest" +"mannWhitneyUTestArgMax" +"mannWhitneyUTestArgMin" +"mannWhitneyUTestArray" +"mannWhitneyUTestDistinct" +"mannWhitneyUTestForEach" +"mannWhitneyUTestIf" +"mannWhitneyUTestMap" +"mannWhitneyUTestMerge" +"mannWhitneyUTestNull" +"mannWhitneyUTestOrDefault" +"mannWhitneyUTestOrNull" +"mannWhitneyUTestResample" +"mannWhitneyUTestSimpleState" +"mannWhitneyUTestState" +"map" +"mapAdd" +"mapAll" +"mapApply" +"mapConcat" +"mapContains" +"mapContainsKeyLike" +"mapExists" +"mapExtractKeyLike" +"mapFilter" +"mapFromArrays" +"mapFromString" +"mapKeys" +"mapPartialReverseSort" +"mapPartialSort" +"mapPopulateSeries" +"mapReverseSort" +"mapSort" +"mapSubtract" +"mapUpdate" +"mapValues" +"match" +"materialize" +"max" +"max2" +"maxArgMax" +"maxArgMin" +"maxArray" +"maxDistinct" +"maxForEach" +"maxIf" +"maxIntersections" +"maxIntersectionsArgMax" +"maxIntersectionsArgMin" +"maxIntersectionsArray" +"maxIntersectionsDistinct" +"maxIntersectionsForEach" +"maxIntersectionsIf" +"maxIntersectionsMap" +"maxIntersectionsMerge" +"maxIntersectionsNull" +"maxIntersectionsOrDefault" +"maxIntersectionsOrNull" +"maxIntersectionsPosition" +"maxIntersectionsPositionArgMax" +"maxIntersectionsPositionArgMin" +"maxIntersectionsPositionArray" +"maxIntersectionsPositionDistinct" +"maxIntersectionsPositionForEach" +"maxIntersectionsPositionIf" +"maxIntersectionsPositionMap" +"maxIntersectionsPositionMerge" +"maxIntersectionsPositionNull" +"maxIntersectionsPositionOrDefault" +"maxIntersectionsPositionOrNull" +"maxIntersectionsPositionResample" +"maxIntersectionsPositionSimpleState" +"maxIntersectionsPositionState" +"maxIntersectionsResample" +"maxIntersectionsSimpleState" +"maxIntersectionsState" +"maxMap" +"maxMappedArrays" +"maxMappedArraysArgMax" +"maxMappedArraysArgMin" +"maxMappedArraysArray" +"maxMappedArraysDistinct" +"maxMappedArraysForEach" +"maxMappedArraysIf" +"maxMappedArraysMap" +"maxMappedArraysMerge" +"maxMappedArraysNull" +"maxMappedArraysOrDefault" +"maxMappedArraysOrNull" +"maxMappedArraysResample" +"maxMappedArraysSimpleState" +"maxMappedArraysState" +"maxMerge" +"maxNull" +"maxOrDefault" +"maxOrNull" +"maxResample" +"maxSimpleState" +"maxState" +"meanZTest" +"meanZTestArgMax" +"meanZTestArgMin" +"meanZTestArray" +"meanZTestDistinct" +"meanZTestForEach" +"meanZTestIf" +"meanZTestMap" +"meanZTestMerge" +"meanZTestNull" +"meanZTestOrDefault" +"meanZTestOrNull" +"meanZTestResample" +"meanZTestSimpleState" +"meanZTestState" +"median" +"medianArgMax" +"medianArgMin" +"medianArray" +"medianBFloat16" +"medianBFloat16ArgMax" +"medianBFloat16ArgMin" +"medianBFloat16Array" +"medianBFloat16Distinct" +"medianBFloat16ForEach" +"medianBFloat16If" +"medianBFloat16Map" +"medianBFloat16Merge" +"medianBFloat16Null" +"medianBFloat16OrDefault" +"medianBFloat16OrNull" +"medianBFloat16Resample" +"medianBFloat16SimpleState" +"medianBFloat16State" +"medianBFloat16Weighted" +"medianBFloat16WeightedArgMax" +"medianBFloat16WeightedArgMin" +"medianBFloat16WeightedArray" +"medianBFloat16WeightedDistinct" +"medianBFloat16WeightedForEach" +"medianBFloat16WeightedIf" +"medianBFloat16WeightedMap" +"medianBFloat16WeightedMerge" +"medianBFloat16WeightedNull" +"medianBFloat16WeightedOrDefault" +"medianBFloat16WeightedOrNull" +"medianBFloat16WeightedResample" +"medianBFloat16WeightedSimpleState" +"medianBFloat16WeightedState" +"medianDD" +"medianDDArgMax" +"medianDDArgMin" +"medianDDArray" +"medianDDDistinct" +"medianDDForEach" +"medianDDIf" +"medianDDMap" +"medianDDMerge" +"medianDDNull" +"medianDDOrDefault" +"medianDDOrNull" +"medianDDResample" +"medianDDSimpleState" +"medianDDState" +"medianDeterministic" +"medianDeterministicArgMax" +"medianDeterministicArgMin" +"medianDeterministicArray" +"medianDeterministicDistinct" +"medianDeterministicForEach" +"medianDeterministicIf" +"medianDeterministicMap" +"medianDeterministicMerge" +"medianDeterministicNull" +"medianDeterministicOrDefault" +"medianDeterministicOrNull" +"medianDeterministicResample" +"medianDeterministicSimpleState" +"medianDeterministicState" +"medianDistinct" +"medianExact" +"medianExactArgMax" +"medianExactArgMin" +"medianExactArray" +"medianExactDistinct" +"medianExactForEach" +"medianExactHigh" +"medianExactHighArgMax" +"medianExactHighArgMin" +"medianExactHighArray" +"medianExactHighDistinct" +"medianExactHighForEach" +"medianExactHighIf" +"medianExactHighMap" +"medianExactHighMerge" +"medianExactHighNull" +"medianExactHighOrDefault" +"medianExactHighOrNull" +"medianExactHighResample" +"medianExactHighSimpleState" +"medianExactHighState" +"medianExactIf" +"medianExactLow" +"medianExactLowArgMax" +"medianExactLowArgMin" +"medianExactLowArray" +"medianExactLowDistinct" +"medianExactLowForEach" +"medianExactLowIf" +"medianExactLowMap" +"medianExactLowMerge" +"medianExactLowNull" +"medianExactLowOrDefault" +"medianExactLowOrNull" +"medianExactLowResample" +"medianExactLowSimpleState" +"medianExactLowState" +"medianExactMap" +"medianExactMerge" +"medianExactNull" +"medianExactOrDefault" +"medianExactOrNull" +"medianExactResample" +"medianExactSimpleState" +"medianExactState" +"medianExactWeighted" +"medianExactWeightedArgMax" +"medianExactWeightedArgMin" +"medianExactWeightedArray" +"medianExactWeightedDistinct" +"medianExactWeightedForEach" +"medianExactWeightedIf" +"medianExactWeightedMap" +"medianExactWeightedMerge" +"medianExactWeightedNull" +"medianExactWeightedOrDefault" +"medianExactWeightedOrNull" +"medianExactWeightedResample" +"medianExactWeightedSimpleState" +"medianExactWeightedState" +"medianForEach" +"medianGK" +"medianGKArgMax" +"medianGKArgMin" +"medianGKArray" +"medianGKDistinct" +"medianGKForEach" +"medianGKIf" +"medianGKMap" +"medianGKMerge" +"medianGKNull" +"medianGKOrDefault" +"medianGKOrNull" +"medianGKResample" +"medianGKSimpleState" +"medianGKState" +"medianIf" +"medianInterpolatedWeighted" +"medianInterpolatedWeightedArgMax" +"medianInterpolatedWeightedArgMin" +"medianInterpolatedWeightedArray" +"medianInterpolatedWeightedDistinct" +"medianInterpolatedWeightedForEach" +"medianInterpolatedWeightedIf" +"medianInterpolatedWeightedMap" +"medianInterpolatedWeightedMerge" +"medianInterpolatedWeightedNull" +"medianInterpolatedWeightedOrDefault" +"medianInterpolatedWeightedOrNull" +"medianInterpolatedWeightedResample" +"medianInterpolatedWeightedSimpleState" +"medianInterpolatedWeightedState" +"medianMap" +"medianMerge" +"medianNull" +"medianOrDefault" +"medianOrNull" +"medianResample" +"medianSimpleState" +"medianState" +"medianTDigest" +"medianTDigestArgMax" +"medianTDigestArgMin" +"medianTDigestArray" +"medianTDigestDistinct" +"medianTDigestForEach" +"medianTDigestIf" +"medianTDigestMap" +"medianTDigestMerge" +"medianTDigestNull" +"medianTDigestOrDefault" +"medianTDigestOrNull" +"medianTDigestResample" +"medianTDigestSimpleState" +"medianTDigestState" +"medianTDigestWeighted" +"medianTDigestWeightedArgMax" +"medianTDigestWeightedArgMin" +"medianTDigestWeightedArray" +"medianTDigestWeightedDistinct" +"medianTDigestWeightedForEach" +"medianTDigestWeightedIf" +"medianTDigestWeightedMap" +"medianTDigestWeightedMerge" +"medianTDigestWeightedNull" +"medianTDigestWeightedOrDefault" +"medianTDigestWeightedOrNull" +"medianTDigestWeightedResample" +"medianTDigestWeightedSimpleState" +"medianTDigestWeightedState" +"medianTiming" +"medianTimingArgMax" +"medianTimingArgMin" +"medianTimingArray" +"medianTimingDistinct" +"medianTimingForEach" +"medianTimingIf" +"medianTimingMap" +"medianTimingMerge" +"medianTimingNull" +"medianTimingOrDefault" +"medianTimingOrNull" +"medianTimingResample" +"medianTimingSimpleState" +"medianTimingState" +"medianTimingWeighted" +"medianTimingWeightedArgMax" +"medianTimingWeightedArgMin" +"medianTimingWeightedArray" +"medianTimingWeightedDistinct" +"medianTimingWeightedForEach" +"medianTimingWeightedIf" +"medianTimingWeightedMap" +"medianTimingWeightedMerge" +"medianTimingWeightedNull" +"medianTimingWeightedOrDefault" +"medianTimingWeightedOrNull" +"medianTimingWeightedResample" +"medianTimingWeightedSimpleState" +"medianTimingWeightedState" +"metroHash64" +"mid" +"min" +"min2" +"minArgMax" +"minArgMin" +"minArray" +"minDistinct" +"minForEach" +"minIf" +"minMap" +"minMappedArrays" +"minMappedArraysArgMax" +"minMappedArraysArgMin" +"minMappedArraysArray" +"minMappedArraysDistinct" +"minMappedArraysForEach" +"minMappedArraysIf" +"minMappedArraysMap" +"minMappedArraysMerge" +"minMappedArraysNull" +"minMappedArraysOrDefault" +"minMappedArraysOrNull" +"minMappedArraysResample" +"minMappedArraysSimpleState" +"minMappedArraysState" +"minMerge" +"minNull" +"minOrDefault" +"minOrNull" +"minResample" +"minSampleSizeContinous" +"minSampleSizeContinuous" +"minSampleSizeConversion" +"minSimpleState" +"minState" +"minus" +"mismatches" +"mod" +"modulo" +"moduloLegacy" +"moduloOrZero" +"monthName" +"mortonDecode" +"mortonEncode" +"multiFuzzyMatchAllIndices" +"multiFuzzyMatchAny" +"multiFuzzyMatchAnyIndex" +"multiIf" +"multiMatchAllIndices" +"multiMatchAny" +"multiMatchAnyIndex" +"multiSearchAllPositions" +"multiSearchAllPositionsCaseInsensitive" +"multiSearchAllPositionsCaseInsensitiveUTF8" +"multiSearchAllPositionsUTF8" +"multiSearchAny" +"multiSearchAnyCaseInsensitive" +"multiSearchAnyCaseInsensitiveUTF8" +"multiSearchAnyUTF8" +"multiSearchFirstIndex" +"multiSearchFirstIndexCaseInsensitive" +"multiSearchFirstIndexCaseInsensitiveUTF8" +"multiSearchFirstIndexUTF8" +"multiSearchFirstPosition" +"multiSearchFirstPositionCaseInsensitive" +"multiSearchFirstPositionCaseInsensitiveUTF8" +"multiSearchFirstPositionUTF8" +"multiply" +"multiplyDecimal" +"murmurHash2_32" +"murmurHash2_64" +"murmurHash3_128" +"murmurHash3_32" +"murmurHash3_64" +"negate" +"neighbor" +"nested" +"netloc" +"ngramDistance" +"ngramDistanceCaseInsensitive" +"ngramDistanceCaseInsensitiveUTF8" +"ngramDistanceUTF8" +"ngramMinHash" +"ngramMinHashArg" +"ngramMinHashArgCaseInsensitive" +"ngramMinHashArgCaseInsensitiveUTF8" +"ngramMinHashArgUTF8" +"ngramMinHashCaseInsensitive" +"ngramMinHashCaseInsensitiveUTF8" +"ngramMinHashUTF8" +"ngramSearch" +"ngramSearchCaseInsensitive" +"ngramSearchCaseInsensitiveUTF8" +"ngramSearchUTF8" +"ngramSimHash" +"ngramSimHashCaseInsensitive" +"ngramSimHashCaseInsensitiveUTF8" +"ngramSimHashUTF8" +"ngrams" +"nonNegativeDerivative" +"nonNegativeDerivativeArgMax" +"nonNegativeDerivativeArgMin" +"nonNegativeDerivativeArray" +"nonNegativeDerivativeDistinct" +"nonNegativeDerivativeForEach" +"nonNegativeDerivativeIf" +"nonNegativeDerivativeMap" +"nonNegativeDerivativeMerge" +"nonNegativeDerivativeNull" +"nonNegativeDerivativeOrDefault" +"nonNegativeDerivativeOrNull" +"nonNegativeDerivativeResample" +"nonNegativeDerivativeSimpleState" +"nonNegativeDerivativeState" +"normL1" +"normL2" +"normL2Squared" +"normLinf" +"normLp" +"normalizeL1" +"normalizeL2" +"normalizeLinf" +"normalizeLp" +"normalizeQuery" +"normalizeQueryKeepNames" +"normalizeUTF8NFC" +"normalizeUTF8NFD" +"normalizeUTF8NFKC" +"normalizeUTF8NFKD" +"normalizedQueryHash" +"normalizedQueryHashKeepNames" +"not" +"notEmpty" +"notEquals" +"notILike" +"notIn" +"notInIgnoreSet" +"notLike" +"notNullIn" +"notNullInIgnoreSet" +"nothing" +"nothingArgMax" +"nothingArgMin" +"nothingArray" +"nothingDistinct" +"nothingForEach" +"nothingIf" +"nothingMap" +"nothingMerge" +"nothingNull" +"nothingNull" +"nothingNullArgMax" +"nothingNullArgMin" +"nothingNullArray" +"nothingNullDistinct" +"nothingNullForEach" +"nothingNullIf" +"nothingNullMap" +"nothingNullMerge" +"nothingNullNull" +"nothingNullOrDefault" +"nothingNullOrNull" +"nothingNullResample" +"nothingNullSimpleState" +"nothingNullState" +"nothingOrDefault" +"nothingOrNull" +"nothingResample" +"nothingSimpleState" +"nothingState" +"nothingUInt64" +"nothingUInt64ArgMax" +"nothingUInt64ArgMin" +"nothingUInt64Array" +"nothingUInt64Distinct" +"nothingUInt64ForEach" +"nothingUInt64If" +"nothingUInt64Map" +"nothingUInt64Merge" +"nothingUInt64Null" +"nothingUInt64OrDefault" +"nothingUInt64OrNull" +"nothingUInt64Resample" +"nothingUInt64SimpleState" +"nothingUInt64State" +"now" +"now64" +"nowInBlock" +"nth_value" +"nth_valueArgMax" +"nth_valueArgMin" +"nth_valueArray" +"nth_valueDistinct" +"nth_valueForEach" +"nth_valueIf" +"nth_valueMap" +"nth_valueMerge" +"nth_valueNull" +"nth_valueOrDefault" +"nth_valueOrNull" +"nth_valueResample" +"nth_valueSimpleState" +"nth_valueState" +"ntile" +"ntileArgMax" +"ntileArgMin" +"ntileArray" +"ntileDistinct" +"ntileForEach" +"ntileIf" +"ntileMap" +"ntileMerge" +"ntileNull" +"ntileOrDefault" +"ntileOrNull" +"ntileResample" +"ntileSimpleState" +"ntileState" +"nullIf" +"nullIn" +"nullInIgnoreSet" +"or" +"parseDateTime" +"parseDateTime32BestEffort" +"parseDateTime32BestEffortOrNull" +"parseDateTime32BestEffortOrZero" +"parseDateTime64BestEffort" +"parseDateTime64BestEffortOrNull" +"parseDateTime64BestEffortOrZero" +"parseDateTime64BestEffortUS" +"parseDateTime64BestEffortUSOrNull" +"parseDateTime64BestEffortUSOrZero" +"parseDateTimeBestEffort" +"parseDateTimeBestEffortOrNull" +"parseDateTimeBestEffortOrZero" +"parseDateTimeBestEffortUS" +"parseDateTimeBestEffortUSOrNull" +"parseDateTimeBestEffortUSOrZero" +"parseDateTimeInJodaSyntax" +"parseDateTimeInJodaSyntaxOrNull" +"parseDateTimeInJodaSyntaxOrZero" +"parseDateTimeOrNull" +"parseDateTimeOrZero" +"parseTimeDelta" +"partitionId" +"path" +"pathFull" +"pi" +"plus" +"pmod" +"pointInEllipses" +"pointInPolygon" +"polygonAreaCartesian" +"polygonAreaSpherical" +"polygonConvexHullCartesian" +"polygonPerimeterCartesian" +"polygonPerimeterSpherical" +"polygonsDistanceCartesian" +"polygonsDistanceSpherical" +"polygonsEqualsCartesian" +"polygonsIntersectionCartesian" +"polygonsIntersectionSpherical" +"polygonsSymDifferenceCartesian" +"polygonsSymDifferenceSpherical" +"polygonsUnionCartesian" +"polygonsUnionSpherical" +"polygonsWithinCartesian" +"polygonsWithinSpherical" +"port" +"portRFC" +"position" +"positionCaseInsensitive" +"positionCaseInsensitiveUTF8" +"positionUTF8" +"positiveModulo" +"positive_modulo" +"pow" +"power" +"proportionsZTest" +"protocol" +"punycodeDecode" +"punycodeEncode" "quantile" +"quantileArgMax" +"quantileArgMin" +"quantileArray" +"quantileBFloat16" +"quantileBFloat16ArgMax" +"quantileBFloat16ArgMin" +"quantileBFloat16Array" +"quantileBFloat16Distinct" +"quantileBFloat16ForEach" +"quantileBFloat16If" +"quantileBFloat16Map" +"quantileBFloat16Merge" +"quantileBFloat16Null" +"quantileBFloat16OrDefault" +"quantileBFloat16OrNull" +"quantileBFloat16Resample" +"quantileBFloat16SimpleState" +"quantileBFloat16State" +"quantileBFloat16Weighted" +"quantileBFloat16WeightedArgMax" +"quantileBFloat16WeightedArgMin" +"quantileBFloat16WeightedArray" +"quantileBFloat16WeightedDistinct" +"quantileBFloat16WeightedForEach" +"quantileBFloat16WeightedIf" +"quantileBFloat16WeightedMap" +"quantileBFloat16WeightedMerge" +"quantileBFloat16WeightedNull" +"quantileBFloat16WeightedOrDefault" +"quantileBFloat16WeightedOrNull" +"quantileBFloat16WeightedResample" +"quantileBFloat16WeightedSimpleState" +"quantileBFloat16WeightedState" +"quantileDD" +"quantileDDArgMax" +"quantileDDArgMin" +"quantileDDArray" +"quantileDDDistinct" +"quantileDDForEach" +"quantileDDIf" +"quantileDDMap" +"quantileDDMerge" +"quantileDDNull" +"quantileDDOrDefault" +"quantileDDOrNull" +"quantileDDResample" +"quantileDDSimpleState" +"quantileDDState" +"quantileDeterministic" +"quantileDeterministicArgMax" +"quantileDeterministicArgMin" +"quantileDeterministicArray" +"quantileDeterministicDistinct" +"quantileDeterministicForEach" +"quantileDeterministicIf" +"quantileDeterministicMap" +"quantileDeterministicMerge" +"quantileDeterministicNull" +"quantileDeterministicOrDefault" +"quantileDeterministicOrNull" +"quantileDeterministicResample" +"quantileDeterministicSimpleState" +"quantileDeterministicState" +"quantileDistinct" +"quantileExact" +"quantileExactArgMax" +"quantileExactArgMin" +"quantileExactArray" +"quantileExactDistinct" +"quantileExactExclusive" +"quantileExactExclusiveArgMax" +"quantileExactExclusiveArgMin" +"quantileExactExclusiveArray" +"quantileExactExclusiveDistinct" +"quantileExactExclusiveForEach" +"quantileExactExclusiveIf" +"quantileExactExclusiveMap" +"quantileExactExclusiveMerge" +"quantileExactExclusiveNull" +"quantileExactExclusiveOrDefault" +"quantileExactExclusiveOrNull" +"quantileExactExclusiveResample" +"quantileExactExclusiveSimpleState" +"quantileExactExclusiveState" +"quantileExactForEach" +"quantileExactHigh" +"quantileExactHighArgMax" +"quantileExactHighArgMin" +"quantileExactHighArray" +"quantileExactHighDistinct" +"quantileExactHighForEach" +"quantileExactHighIf" +"quantileExactHighMap" +"quantileExactHighMerge" +"quantileExactHighNull" +"quantileExactHighOrDefault" +"quantileExactHighOrNull" +"quantileExactHighResample" +"quantileExactHighSimpleState" +"quantileExactHighState" +"quantileExactIf" +"quantileExactInclusive" +"quantileExactInclusiveArgMax" +"quantileExactInclusiveArgMin" +"quantileExactInclusiveArray" +"quantileExactInclusiveDistinct" +"quantileExactInclusiveForEach" +"quantileExactInclusiveIf" +"quantileExactInclusiveMap" +"quantileExactInclusiveMerge" +"quantileExactInclusiveNull" +"quantileExactInclusiveOrDefault" +"quantileExactInclusiveOrNull" +"quantileExactInclusiveResample" +"quantileExactInclusiveSimpleState" +"quantileExactInclusiveState" +"quantileExactLow" +"quantileExactLowArgMax" +"quantileExactLowArgMin" +"quantileExactLowArray" +"quantileExactLowDistinct" +"quantileExactLowForEach" +"quantileExactLowIf" +"quantileExactLowMap" +"quantileExactLowMerge" +"quantileExactLowNull" +"quantileExactLowOrDefault" +"quantileExactLowOrNull" +"quantileExactLowResample" +"quantileExactLowSimpleState" +"quantileExactLowState" +"quantileExactMap" +"quantileExactMerge" +"quantileExactNull" +"quantileExactOrDefault" +"quantileExactOrNull" +"quantileExactResample" +"quantileExactSimpleState" +"quantileExactState" +"quantileExactWeighted" +"quantileExactWeightedArgMax" +"quantileExactWeightedArgMin" +"quantileExactWeightedArray" +"quantileExactWeightedDistinct" +"quantileExactWeightedForEach" +"quantileExactWeightedIf" +"quantileExactWeightedMap" +"quantileExactWeightedMerge" +"quantileExactWeightedNull" +"quantileExactWeightedOrDefault" +"quantileExactWeightedOrNull" +"quantileExactWeightedResample" +"quantileExactWeightedSimpleState" +"quantileExactWeightedState" +"quantileForEach" +"quantileGK" +"quantileGKArgMax" +"quantileGKArgMin" +"quantileGKArray" +"quantileGKDistinct" +"quantileGKForEach" +"quantileGKIf" +"quantileGKMap" +"quantileGKMerge" +"quantileGKNull" +"quantileGKOrDefault" +"quantileGKOrNull" +"quantileGKResample" +"quantileGKSimpleState" +"quantileGKState" +"quantileIf" +"quantileInterpolatedWeighted" +"quantileInterpolatedWeightedArgMax" +"quantileInterpolatedWeightedArgMin" +"quantileInterpolatedWeightedArray" +"quantileInterpolatedWeightedDistinct" +"quantileInterpolatedWeightedForEach" +"quantileInterpolatedWeightedIf" +"quantileInterpolatedWeightedMap" +"quantileInterpolatedWeightedMerge" +"quantileInterpolatedWeightedNull" +"quantileInterpolatedWeightedOrDefault" +"quantileInterpolatedWeightedOrNull" +"quantileInterpolatedWeightedResample" +"quantileInterpolatedWeightedSimpleState" +"quantileInterpolatedWeightedState" +"quantileMap" +"quantileMerge" +"quantileNull" +"quantileOrDefault" +"quantileOrNull" +"quantileResample" +"quantileSimpleState" +"quantileState" +"quantileTDigest" +"quantileTDigestArgMax" +"quantileTDigestArgMin" +"quantileTDigestArray" +"quantileTDigestDistinct" +"quantileTDigestForEach" +"quantileTDigestIf" +"quantileTDigestMap" +"quantileTDigestMerge" +"quantileTDigestNull" +"quantileTDigestOrDefault" +"quantileTDigestOrNull" +"quantileTDigestResample" +"quantileTDigestSimpleState" +"quantileTDigestState" +"quantileTDigestWeighted" +"quantileTDigestWeightedArgMax" +"quantileTDigestWeightedArgMin" +"quantileTDigestWeightedArray" +"quantileTDigestWeightedDistinct" +"quantileTDigestWeightedForEach" +"quantileTDigestWeightedIf" +"quantileTDigestWeightedMap" +"quantileTDigestWeightedMerge" +"quantileTDigestWeightedNull" +"quantileTDigestWeightedOrDefault" +"quantileTDigestWeightedOrNull" +"quantileTDigestWeightedResample" +"quantileTDigestWeightedSimpleState" +"quantileTDigestWeightedState" +"quantileTiming" +"quantileTimingArgMax" +"quantileTimingArgMin" +"quantileTimingArray" +"quantileTimingDistinct" +"quantileTimingForEach" +"quantileTimingIf" +"quantileTimingMap" +"quantileTimingMerge" +"quantileTimingNull" +"quantileTimingOrDefault" +"quantileTimingOrNull" +"quantileTimingResample" +"quantileTimingSimpleState" +"quantileTimingState" +"quantileTimingWeighted" +"quantileTimingWeightedArgMax" +"quantileTimingWeightedArgMin" +"quantileTimingWeightedArray" +"quantileTimingWeightedDistinct" +"quantileTimingWeightedForEach" +"quantileTimingWeightedIf" +"quantileTimingWeightedMap" +"quantileTimingWeightedMerge" +"quantileTimingWeightedNull" +"quantileTimingWeightedOrDefault" +"quantileTimingWeightedOrNull" +"quantileTimingWeightedResample" +"quantileTimingWeightedSimpleState" +"quantileTimingWeightedState" +"quantiles" +"quantilesArgMax" +"quantilesArgMin" +"quantilesArray" +"quantilesBFloat16" +"quantilesBFloat16ArgMax" +"quantilesBFloat16ArgMin" +"quantilesBFloat16Array" +"quantilesBFloat16Distinct" +"quantilesBFloat16ForEach" +"quantilesBFloat16If" +"quantilesBFloat16Map" +"quantilesBFloat16Merge" +"quantilesBFloat16Null" +"quantilesBFloat16OrDefault" +"quantilesBFloat16OrNull" +"quantilesBFloat16Resample" +"quantilesBFloat16SimpleState" +"quantilesBFloat16State" +"quantilesBFloat16Weighted" +"quantilesBFloat16WeightedArgMax" +"quantilesBFloat16WeightedArgMin" +"quantilesBFloat16WeightedArray" +"quantilesBFloat16WeightedDistinct" +"quantilesBFloat16WeightedForEach" +"quantilesBFloat16WeightedIf" +"quantilesBFloat16WeightedMap" +"quantilesBFloat16WeightedMerge" +"quantilesBFloat16WeightedNull" +"quantilesBFloat16WeightedOrDefault" +"quantilesBFloat16WeightedOrNull" +"quantilesBFloat16WeightedResample" +"quantilesBFloat16WeightedSimpleState" +"quantilesBFloat16WeightedState" +"quantilesDD" +"quantilesDDArgMax" +"quantilesDDArgMin" +"quantilesDDArray" +"quantilesDDDistinct" +"quantilesDDForEach" +"quantilesDDIf" +"quantilesDDMap" +"quantilesDDMerge" +"quantilesDDNull" +"quantilesDDOrDefault" +"quantilesDDOrNull" +"quantilesDDResample" +"quantilesDDSimpleState" +"quantilesDDState" +"quantilesDeterministic" +"quantilesDeterministicArgMax" +"quantilesDeterministicArgMin" +"quantilesDeterministicArray" +"quantilesDeterministicDistinct" +"quantilesDeterministicForEach" +"quantilesDeterministicIf" +"quantilesDeterministicMap" +"quantilesDeterministicMerge" +"quantilesDeterministicNull" +"quantilesDeterministicOrDefault" +"quantilesDeterministicOrNull" +"quantilesDeterministicResample" +"quantilesDeterministicSimpleState" +"quantilesDeterministicState" +"quantilesDistinct" +"quantilesExact" +"quantilesExactArgMax" +"quantilesExactArgMin" +"quantilesExactArray" +"quantilesExactDistinct" +"quantilesExactExclusive" +"quantilesExactExclusiveArgMax" +"quantilesExactExclusiveArgMin" +"quantilesExactExclusiveArray" +"quantilesExactExclusiveDistinct" +"quantilesExactExclusiveForEach" +"quantilesExactExclusiveIf" +"quantilesExactExclusiveMap" +"quantilesExactExclusiveMerge" +"quantilesExactExclusiveNull" +"quantilesExactExclusiveOrDefault" +"quantilesExactExclusiveOrNull" +"quantilesExactExclusiveResample" +"quantilesExactExclusiveSimpleState" +"quantilesExactExclusiveState" +"quantilesExactForEach" +"quantilesExactHigh" +"quantilesExactHighArgMax" +"quantilesExactHighArgMin" +"quantilesExactHighArray" +"quantilesExactHighDistinct" +"quantilesExactHighForEach" +"quantilesExactHighIf" +"quantilesExactHighMap" +"quantilesExactHighMerge" +"quantilesExactHighNull" +"quantilesExactHighOrDefault" +"quantilesExactHighOrNull" +"quantilesExactHighResample" +"quantilesExactHighSimpleState" +"quantilesExactHighState" +"quantilesExactIf" +"quantilesExactInclusive" +"quantilesExactInclusiveArgMax" +"quantilesExactInclusiveArgMin" +"quantilesExactInclusiveArray" +"quantilesExactInclusiveDistinct" +"quantilesExactInclusiveForEach" +"quantilesExactInclusiveIf" +"quantilesExactInclusiveMap" +"quantilesExactInclusiveMerge" +"quantilesExactInclusiveNull" +"quantilesExactInclusiveOrDefault" +"quantilesExactInclusiveOrNull" +"quantilesExactInclusiveResample" +"quantilesExactInclusiveSimpleState" +"quantilesExactInclusiveState" +"quantilesExactLow" +"quantilesExactLowArgMax" +"quantilesExactLowArgMin" +"quantilesExactLowArray" +"quantilesExactLowDistinct" +"quantilesExactLowForEach" +"quantilesExactLowIf" +"quantilesExactLowMap" +"quantilesExactLowMerge" +"quantilesExactLowNull" +"quantilesExactLowOrDefault" +"quantilesExactLowOrNull" +"quantilesExactLowResample" +"quantilesExactLowSimpleState" +"quantilesExactLowState" +"quantilesExactMap" +"quantilesExactMerge" +"quantilesExactNull" +"quantilesExactOrDefault" +"quantilesExactOrNull" +"quantilesExactResample" +"quantilesExactSimpleState" +"quantilesExactState" +"quantilesExactWeighted" +"quantilesExactWeightedArgMax" +"quantilesExactWeightedArgMin" +"quantilesExactWeightedArray" +"quantilesExactWeightedDistinct" +"quantilesExactWeightedForEach" +"quantilesExactWeightedIf" +"quantilesExactWeightedMap" +"quantilesExactWeightedMerge" +"quantilesExactWeightedNull" +"quantilesExactWeightedOrDefault" +"quantilesExactWeightedOrNull" +"quantilesExactWeightedResample" +"quantilesExactWeightedSimpleState" +"quantilesExactWeightedState" +"quantilesForEach" +"quantilesGK" +"quantilesGKArgMax" +"quantilesGKArgMin" +"quantilesGKArray" +"quantilesGKDistinct" +"quantilesGKForEach" +"quantilesGKIf" +"quantilesGKMap" +"quantilesGKMerge" +"quantilesGKNull" +"quantilesGKOrDefault" +"quantilesGKOrNull" +"quantilesGKResample" +"quantilesGKSimpleState" +"quantilesGKState" +"quantilesIf" +"quantilesInterpolatedWeighted" +"quantilesInterpolatedWeightedArgMax" +"quantilesInterpolatedWeightedArgMin" +"quantilesInterpolatedWeightedArray" +"quantilesInterpolatedWeightedDistinct" +"quantilesInterpolatedWeightedForEach" +"quantilesInterpolatedWeightedIf" +"quantilesInterpolatedWeightedMap" +"quantilesInterpolatedWeightedMerge" +"quantilesInterpolatedWeightedNull" +"quantilesInterpolatedWeightedOrDefault" +"quantilesInterpolatedWeightedOrNull" +"quantilesInterpolatedWeightedResample" +"quantilesInterpolatedWeightedSimpleState" +"quantilesInterpolatedWeightedState" +"quantilesMap" +"quantilesMerge" +"quantilesNull" +"quantilesOrDefault" +"quantilesOrNull" +"quantilesResample" +"quantilesSimpleState" +"quantilesState" +"quantilesTDigest" +"quantilesTDigestArgMax" +"quantilesTDigestArgMin" +"quantilesTDigestArray" +"quantilesTDigestDistinct" +"quantilesTDigestForEach" +"quantilesTDigestIf" +"quantilesTDigestMap" +"quantilesTDigestMerge" +"quantilesTDigestNull" +"quantilesTDigestOrDefault" +"quantilesTDigestOrNull" +"quantilesTDigestResample" +"quantilesTDigestSimpleState" +"quantilesTDigestState" +"quantilesTDigestWeighted" +"quantilesTDigestWeightedArgMax" +"quantilesTDigestWeightedArgMin" +"quantilesTDigestWeightedArray" +"quantilesTDigestWeightedDistinct" +"quantilesTDigestWeightedForEach" +"quantilesTDigestWeightedIf" +"quantilesTDigestWeightedMap" +"quantilesTDigestWeightedMerge" +"quantilesTDigestWeightedNull" +"quantilesTDigestWeightedOrDefault" +"quantilesTDigestWeightedOrNull" +"quantilesTDigestWeightedResample" +"quantilesTDigestWeightedSimpleState" +"quantilesTDigestWeightedState" +"quantilesTiming" +"quantilesTimingArgMax" +"quantilesTimingArgMin" +"quantilesTimingArray" +"quantilesTimingDistinct" +"quantilesTimingForEach" +"quantilesTimingIf" +"quantilesTimingMap" +"quantilesTimingMerge" +"quantilesTimingNull" +"quantilesTimingOrDefault" +"quantilesTimingOrNull" +"quantilesTimingResample" +"quantilesTimingSimpleState" +"quantilesTimingState" +"quantilesTimingWeighted" +"quantilesTimingWeightedArgMax" +"quantilesTimingWeightedArgMin" +"quantilesTimingWeightedArray" +"quantilesTimingWeightedDistinct" +"quantilesTimingWeightedForEach" +"quantilesTimingWeightedIf" +"quantilesTimingWeightedMap" +"quantilesTimingWeightedMerge" +"quantilesTimingWeightedNull" +"quantilesTimingWeightedOrDefault" +"quantilesTimingWeightedOrNull" +"quantilesTimingWeightedResample" +"quantilesTimingWeightedSimpleState" +"quantilesTimingWeightedState" +"queryID" +"queryString" +"queryStringAndFragment" +"query_id" +"radians" +"rand" +"rand32" +"rand64" +"randBernoulli" +"randBinomial" +"randCanonical" +"randChiSquared" +"randConstant" +"randExponential" +"randFisherF" +"randLogNormal" +"randNegativeBinomial" +"randNormal" +"randPoisson" +"randStudentT" +"randUniform" +"randomFixedString" +"randomPrintableASCII" +"randomString" +"randomStringUTF8" +"range" +"rank" +"rankArgMax" +"rankArgMin" +"rankArray" +"rankCorr" +"rankCorrArgMax" +"rankCorrArgMin" +"rankCorrArray" +"rankCorrDistinct" +"rankCorrForEach" +"rankCorrIf" +"rankCorrMap" +"rankCorrMerge" +"rankCorrNull" +"rankCorrOrDefault" +"rankCorrOrNull" +"rankCorrResample" +"rankCorrSimpleState" +"rankCorrState" +"rankDistinct" +"rankForEach" +"rankIf" +"rankMap" +"rankMerge" +"rankNull" +"rankOrDefault" +"rankOrNull" +"rankResample" +"rankSimpleState" +"rankState" +"readWKTMultiPolygon" +"readWKTPoint" +"readWKTPolygon" +"readWKTRing" +"regexpExtract" +"regexpQuoteMeta" +"regionHierarchy" +"regionIn" +"regionToArea" +"regionToCity" +"regionToContinent" +"regionToCountry" +"regionToDistrict" +"regionToName" +"regionToPopulation" +"regionToTopContinent" +"reinterpret" +"reinterpretAsDate" +"reinterpretAsDateTime" +"reinterpretAsFixedString" +"reinterpretAsFloat32" +"reinterpretAsFloat64" +"reinterpretAsInt128" +"reinterpretAsInt16" +"reinterpretAsInt256" +"reinterpretAsInt32" +"reinterpretAsInt64" +"reinterpretAsInt8" +"reinterpretAsString" +"reinterpretAsUInt128" +"reinterpretAsUInt16" +"reinterpretAsUInt256" +"reinterpretAsUInt32" +"reinterpretAsUInt64" +"reinterpretAsUInt8" +"reinterpretAsUUID" +"repeat" +"replace" +"replaceAll" +"replaceOne" +"replaceRegexpAll" +"replaceRegexpOne" +"replicate" +"retention" +"retentionArgMax" +"retentionArgMin" +"retentionArray" +"retentionDistinct" +"retentionForEach" +"retentionIf" +"retentionMap" +"retentionMerge" +"retentionNull" +"retentionOrDefault" +"retentionOrNull" +"retentionResample" +"retentionSimpleState" +"retentionState" +"reverse" +"reverseUTF8" +"revision" +"right" +"rightPad" +"rightPadUTF8" +"rightUTF8" +"round" +"roundAge" +"roundBankers" +"roundDown" +"roundDuration" +"roundToExp2" +"rowNumberInAllBlocks" +"rowNumberInBlock" +"row_number" +"row_numberArgMax" +"row_numberArgMin" +"row_numberArray" +"row_numberDistinct" +"row_numberForEach" +"row_numberIf" +"row_numberMap" +"row_numberMerge" +"row_numberNull" +"row_numberOrDefault" +"row_numberOrNull" +"row_numberResample" +"row_numberSimpleState" +"row_numberState" +"rpad" +"rtrim" +"runningAccumulate" +"runningConcurrency" +"runningDifference" +"runningDifferenceStartingWithFirstValue" +"s2CapContains" +"s2CapUnion" +"s2CellsIntersect" +"s2GetNeighbors" +"s2RectAdd" +"s2RectContains" +"s2RectIntersection" +"s2RectUnion" +"s2ToGeo" +"scalarProduct" +"sequenceCount" +"sequenceCountArgMax" +"sequenceCountArgMin" +"sequenceCountArray" +"sequenceCountDistinct" +"sequenceCountForEach" +"sequenceCountIf" +"sequenceCountMap" +"sequenceCountMerge" +"sequenceCountNull" +"sequenceCountOrDefault" +"sequenceCountOrNull" +"sequenceCountResample" +"sequenceCountSimpleState" +"sequenceCountState" +"sequenceMatch" +"sequenceMatchArgMax" +"sequenceMatchArgMin" +"sequenceMatchArray" +"sequenceMatchDistinct" +"sequenceMatchForEach" +"sequenceMatchIf" +"sequenceMatchMap" +"sequenceMatchMerge" +"sequenceMatchNull" +"sequenceMatchOrDefault" +"sequenceMatchOrNull" +"sequenceMatchResample" +"sequenceMatchSimpleState" +"sequenceMatchState" +"sequenceNextNode" +"sequenceNextNodeArgMax" +"sequenceNextNodeArgMin" +"sequenceNextNodeArray" +"sequenceNextNodeDistinct" +"sequenceNextNodeForEach" +"sequenceNextNodeIf" +"sequenceNextNodeMap" +"sequenceNextNodeMerge" +"sequenceNextNodeNull" +"sequenceNextNodeOrDefault" +"sequenceNextNodeOrNull" +"sequenceNextNodeResample" +"sequenceNextNodeSimpleState" +"sequenceNextNodeState" +"seriesDecomposeSTL" +"seriesOutliersDetectTukey" +"seriesPeriodDetectFFT" +"serverTimeZone" +"serverTimezone" +"serverUUID" +"shardCount" +"shardNum" +"showCertificate" +"sigmoid" +"sign" +"simpleJSONExtractBool" +"simpleJSONExtractFloat" +"simpleJSONExtractInt" +"simpleJSONExtractRaw" +"simpleJSONExtractString" +"simpleJSONExtractUInt" +"simpleJSONHas" +"simpleLinearRegression" +"simpleLinearRegressionArgMax" +"simpleLinearRegressionArgMin" +"simpleLinearRegressionArray" +"simpleLinearRegressionDistinct" +"simpleLinearRegressionForEach" +"simpleLinearRegressionIf" +"simpleLinearRegressionMap" +"simpleLinearRegressionMerge" +"simpleLinearRegressionNull" +"simpleLinearRegressionOrDefault" +"simpleLinearRegressionOrNull" +"simpleLinearRegressionResample" +"simpleLinearRegressionSimpleState" +"simpleLinearRegressionState" +"sin" +"singleValueOrNull" +"singleValueOrNullArgMax" +"singleValueOrNullArgMin" +"singleValueOrNullArray" +"singleValueOrNullDistinct" +"singleValueOrNullForEach" +"singleValueOrNullIf" +"singleValueOrNullMap" +"singleValueOrNullMerge" +"singleValueOrNullNull" +"singleValueOrNullOrDefault" +"singleValueOrNullOrNull" +"singleValueOrNullResample" +"singleValueOrNullSimpleState" +"singleValueOrNullState" +"sinh" +"sipHash128" +"sipHash128Keyed" +"sipHash128Reference" +"sipHash128ReferenceKeyed" +"sipHash64" +"sipHash64Keyed" +"skewPop" +"skewPopArgMax" +"skewPopArgMin" +"skewPopArray" +"skewPopDistinct" +"skewPopForEach" +"skewPopIf" +"skewPopMap" +"skewPopMerge" +"skewPopNull" +"skewPopOrDefault" +"skewPopOrNull" +"skewPopResample" +"skewPopSimpleState" +"skewPopState" +"skewSamp" +"skewSampArgMax" +"skewSampArgMin" +"skewSampArray" +"skewSampDistinct" +"skewSampForEach" +"skewSampIf" +"skewSampMap" +"skewSampMerge" +"skewSampNull" +"skewSampOrDefault" +"skewSampOrNull" +"skewSampResample" +"skewSampSimpleState" +"skewSampState" +"sleep" +"sleepEachRow" +"snowflakeToDateTime" +"snowflakeToDateTime64" +"soundex" +"space" +"sparkBar" +"sparkBarArgMax" +"sparkBarArgMin" +"sparkBarArray" +"sparkBarDistinct" +"sparkBarForEach" +"sparkBarIf" +"sparkBarMap" +"sparkBarMerge" +"sparkBarNull" +"sparkBarOrDefault" +"sparkBarOrNull" +"sparkBarResample" +"sparkBarSimpleState" +"sparkBarState" +"sparkbar" +"sparkbarArgMax" +"sparkbarArgMin" +"sparkbarArray" +"sparkbarDistinct" +"sparkbarForEach" +"sparkbarIf" +"sparkbarMap" +"sparkbarMerge" +"sparkbarNull" +"sparkbarOrDefault" +"sparkbarOrNull" +"sparkbarResample" +"sparkbarSimpleState" +"sparkbarState" +"splitByAlpha" +"splitByChar" +"splitByNonAlpha" +"splitByRegexp" +"splitByString" +"splitByWhitespace" +"sqid" +"sqidDecode" +"sqidEncode" +"sqrt" +"startsWith" +"startsWithUTF8" +"stddevPop" +"stddevPopArgMax" +"stddevPopArgMin" +"stddevPopArray" +"stddevPopDistinct" +"stddevPopForEach" +"stddevPopIf" +"stddevPopMap" +"stddevPopMerge" +"stddevPopNull" +"stddevPopOrDefault" +"stddevPopOrNull" +"stddevPopResample" +"stddevPopSimpleState" +"stddevPopStable" +"stddevPopStableArgMax" +"stddevPopStableArgMin" +"stddevPopStableArray" +"stddevPopStableDistinct" +"stddevPopStableForEach" +"stddevPopStableIf" +"stddevPopStableMap" +"stddevPopStableMerge" +"stddevPopStableNull" +"stddevPopStableOrDefault" +"stddevPopStableOrNull" +"stddevPopStableResample" +"stddevPopStableSimpleState" +"stddevPopStableState" +"stddevPopState" +"stddevSamp" +"stddevSampArgMax" +"stddevSampArgMin" +"stddevSampArray" +"stddevSampDistinct" +"stddevSampForEach" +"stddevSampIf" +"stddevSampMap" +"stddevSampMerge" +"stddevSampNull" +"stddevSampOrDefault" +"stddevSampOrNull" +"stddevSampResample" +"stddevSampSimpleState" +"stddevSampStable" +"stddevSampStableArgMax" +"stddevSampStableArgMin" +"stddevSampStableArray" +"stddevSampStableDistinct" +"stddevSampStableForEach" +"stddevSampStableIf" +"stddevSampStableMap" +"stddevSampStableMerge" +"stddevSampStableNull" +"stddevSampStableOrDefault" +"stddevSampStableOrNull" +"stddevSampStableResample" +"stddevSampStableSimpleState" +"stddevSampStableState" +"stddevSampState" +"stem" +"stochasticLinearRegression" +"stochasticLinearRegressionArgMax" +"stochasticLinearRegressionArgMin" +"stochasticLinearRegressionArray" +"stochasticLinearRegressionDistinct" +"stochasticLinearRegressionForEach" +"stochasticLinearRegressionIf" +"stochasticLinearRegressionMap" +"stochasticLinearRegressionMerge" +"stochasticLinearRegressionNull" +"stochasticLinearRegressionOrDefault" +"stochasticLinearRegressionOrNull" +"stochasticLinearRegressionResample" +"stochasticLinearRegressionSimpleState" +"stochasticLinearRegressionState" +"stochasticLogisticRegression" +"stochasticLogisticRegressionArgMax" +"stochasticLogisticRegressionArgMin" +"stochasticLogisticRegressionArray" +"stochasticLogisticRegressionDistinct" +"stochasticLogisticRegressionForEach" +"stochasticLogisticRegressionIf" +"stochasticLogisticRegressionMap" +"stochasticLogisticRegressionMerge" +"stochasticLogisticRegressionNull" +"stochasticLogisticRegressionOrDefault" +"stochasticLogisticRegressionOrNull" +"stochasticLogisticRegressionResample" +"stochasticLogisticRegressionSimpleState" +"stochasticLogisticRegressionState" +"str_to_date" +"str_to_map" +"stringJaccardIndex" +"stringJaccardIndexUTF8" +"stringToH3" +"structureToCapnProtoSchema" +"structureToProtobufSchema" +"studentTTest" +"studentTTestArgMax" +"studentTTestArgMin" +"studentTTestArray" +"studentTTestDistinct" +"studentTTestForEach" +"studentTTestIf" +"studentTTestMap" +"studentTTestMerge" +"studentTTestNull" +"studentTTestOrDefault" +"studentTTestOrNull" +"studentTTestResample" +"studentTTestSimpleState" +"studentTTestState" +"subBitmap" +"subDate" +"substr" +"substring" +"substringIndex" +"substringIndexUTF8" +"substringUTF8" +"subtractDays" +"subtractHours" +"subtractInterval" +"subtractMicroseconds" +"subtractMilliseconds" +"subtractMinutes" +"subtractMonths" +"subtractNanoseconds" +"subtractQuarters" +"subtractSeconds" +"subtractTupleOfIntervals" +"subtractWeeks" +"subtractYears" +"sum" +"sumArgMax" +"sumArgMin" +"sumArray" +"sumCount" +"sumCountArgMax" +"sumCountArgMin" +"sumCountArray" +"sumCountDistinct" +"sumCountForEach" +"sumCountIf" +"sumCountMap" +"sumCountMerge" +"sumCountNull" +"sumCountOrDefault" +"sumCountOrNull" +"sumCountResample" +"sumCountSimpleState" +"sumCountState" +"sumDistinct" +"sumForEach" +"sumIf" +"sumKahan" +"sumKahanArgMax" +"sumKahanArgMin" +"sumKahanArray" +"sumKahanDistinct" +"sumKahanForEach" +"sumKahanIf" +"sumKahanMap" +"sumKahanMerge" +"sumKahanNull" +"sumKahanOrDefault" +"sumKahanOrNull" +"sumKahanResample" +"sumKahanSimpleState" +"sumKahanState" +"sumMap" +"sumMapFiltered" +"sumMapFilteredArgMax" +"sumMapFilteredArgMin" +"sumMapFilteredArray" +"sumMapFilteredDistinct" +"sumMapFilteredForEach" +"sumMapFilteredIf" +"sumMapFilteredMap" +"sumMapFilteredMerge" +"sumMapFilteredNull" +"sumMapFilteredOrDefault" +"sumMapFilteredOrNull" +"sumMapFilteredResample" +"sumMapFilteredSimpleState" +"sumMapFilteredState" +"sumMapFilteredWithOverflow" +"sumMapFilteredWithOverflowArgMax" +"sumMapFilteredWithOverflowArgMin" +"sumMapFilteredWithOverflowArray" +"sumMapFilteredWithOverflowDistinct" +"sumMapFilteredWithOverflowForEach" +"sumMapFilteredWithOverflowIf" +"sumMapFilteredWithOverflowMap" +"sumMapFilteredWithOverflowMerge" +"sumMapFilteredWithOverflowNull" +"sumMapFilteredWithOverflowOrDefault" +"sumMapFilteredWithOverflowOrNull" +"sumMapFilteredWithOverflowResample" +"sumMapFilteredWithOverflowSimpleState" +"sumMapFilteredWithOverflowState" +"sumMapWithOverflow" +"sumMapWithOverflowArgMax" +"sumMapWithOverflowArgMin" +"sumMapWithOverflowArray" +"sumMapWithOverflowDistinct" +"sumMapWithOverflowForEach" +"sumMapWithOverflowIf" +"sumMapWithOverflowMap" +"sumMapWithOverflowMerge" +"sumMapWithOverflowNull" +"sumMapWithOverflowOrDefault" +"sumMapWithOverflowOrNull" +"sumMapWithOverflowResample" +"sumMapWithOverflowSimpleState" +"sumMapWithOverflowState" +"sumMappedArrays" +"sumMappedArraysArgMax" +"sumMappedArraysArgMin" +"sumMappedArraysArray" +"sumMappedArraysDistinct" +"sumMappedArraysForEach" +"sumMappedArraysIf" +"sumMappedArraysMap" +"sumMappedArraysMerge" +"sumMappedArraysNull" +"sumMappedArraysOrDefault" +"sumMappedArraysOrNull" +"sumMappedArraysResample" +"sumMappedArraysSimpleState" +"sumMappedArraysState" +"sumMerge" +"sumNull" +"sumOrDefault" +"sumOrNull" +"sumResample" +"sumSimpleState" +"sumState" +"sumWithOverflow" +"sumWithOverflowArgMax" +"sumWithOverflowArgMin" +"sumWithOverflowArray" +"sumWithOverflowDistinct" +"sumWithOverflowForEach" +"sumWithOverflowIf" +"sumWithOverflowMap" +"sumWithOverflowMerge" +"sumWithOverflowNull" +"sumWithOverflowOrDefault" +"sumWithOverflowOrNull" +"sumWithOverflowResample" +"sumWithOverflowSimpleState" +"sumWithOverflowState" +"svg" +"synonyms" +"tan" +"tanh" +"tcpPort" +"tgamma" +"theilsU" +"theilsUArgMax" +"theilsUArgMin" +"theilsUArray" +"theilsUDistinct" +"theilsUForEach" +"theilsUIf" +"theilsUMap" +"theilsUMerge" +"theilsUNull" +"theilsUOrDefault" +"theilsUOrNull" +"theilsUResample" +"theilsUSimpleState" +"theilsUState" +"throwIf" +"tid" +"timeDiff" +"timeSlot" +"timeSlots" +"timeZone" +"timeZoneOf" +"timeZoneOffset" +"timestamp" +"timestampDiff" +"timestamp_diff" +"timezone" +"timezoneOf" +"timezoneOffset" +"toBool" +"toColumnTypeName" +"toDate" +"toDate32" +"toDate32OrDefault" +"toDate32OrNull" +"toDate32OrZero" +"toDateOrDefault" +"toDateOrNull" +"toDateOrZero" +"toDateTime" +"toDateTime32" +"toDateTime64" +"toDateTime64OrDefault" +"toDateTime64OrNull" +"toDateTime64OrZero" +"toDateTimeOrDefault" +"toDateTimeOrNull" +"toDateTimeOrZero" +"toDayOfMonth" +"toDayOfWeek" +"toDayOfYear" +"toDaysSinceYearZero" +"toDecimal128" +"toDecimal128OrDefault" +"toDecimal128OrNull" +"toDecimal128OrZero" +"toDecimal256" +"toDecimal256OrDefault" +"toDecimal256OrNull" +"toDecimal256OrZero" +"toDecimal32" +"toDecimal32OrDefault" +"toDecimal32OrNull" +"toDecimal32OrZero" +"toDecimal64" +"toDecimal64OrDefault" +"toDecimal64OrNull" +"toDecimal64OrZero" +"toDecimalString" +"toFixedString" +"toFloat32" +"toFloat32OrDefault" +"toFloat32OrNull" +"toFloat32OrZero" +"toFloat64" +"toFloat64OrDefault" +"toFloat64OrNull" +"toFloat64OrZero" +"toHour" +"toIPv4" +"toIPv4OrDefault" +"toIPv4OrNull" +"toIPv4OrZero" +"toIPv6" +"toIPv6OrDefault" +"toIPv6OrNull" +"toIPv6OrZero" +"toISOWeek" +"toISOYear" +"toInt128" +"toInt128OrDefault" +"toInt128OrNull" +"toInt128OrZero" +"toInt16" +"toInt16OrDefault" +"toInt16OrNull" +"toInt16OrZero" +"toInt256" +"toInt256OrDefault" +"toInt256OrNull" +"toInt256OrZero" +"toInt32" +"toInt32OrDefault" +"toInt32OrNull" +"toInt32OrZero" +"toInt64" +"toInt64OrDefault" +"toInt64OrNull" +"toInt64OrZero" +"toInt8" +"toInt8OrDefault" +"toInt8OrNull" +"toInt8OrZero" +"toIntervalDay" +"toIntervalHour" +"toIntervalMicrosecond" +"toIntervalMillisecond" +"toIntervalMinute" +"toIntervalMonth" +"toIntervalNanosecond" +"toIntervalQuarter" +"toIntervalSecond" +"toIntervalWeek" +"toIntervalYear" +"toJSONString" +"toLastDayOfMonth" +"toLastDayOfWeek" +"toLowCardinality" +"toMillisecond" +"toMinute" +"toModifiedJulianDay" +"toModifiedJulianDayOrNull" +"toMonday" +"toMonth" +"toNullable" +"toQuarter" +"toRelativeDayNum" +"toRelativeHourNum" +"toRelativeMinuteNum" +"toRelativeMonthNum" +"toRelativeQuarterNum" +"toRelativeSecondNum" +"toRelativeWeekNum" +"toRelativeYearNum" +"toSecond" +"toStartOfDay" +"toStartOfFifteenMinutes" +"toStartOfFiveMinute" +"toStartOfFiveMinutes" +"toStartOfHour" +"toStartOfISOYear" +"toStartOfInterval" +"toStartOfMicrosecond" +"toStartOfMillisecond" +"toStartOfMinute" +"toStartOfMonth" +"toStartOfNanosecond" +"toStartOfQuarter" +"toStartOfSecond" +"toStartOfTenMinutes" +"toStartOfWeek" +"toStartOfYear" +"toString" +"toStringCutToZero" +"toTime" +"toTimeZone" +"toTimezone" +"toTypeName" +"toUInt128" +"toUInt128OrDefault" +"toUInt128OrNull" +"toUInt128OrZero" +"toUInt16" +"toUInt16OrDefault" +"toUInt16OrNull" +"toUInt16OrZero" +"toUInt256" +"toUInt256OrDefault" +"toUInt256OrNull" +"toUInt256OrZero" +"toUInt32" +"toUInt32OrDefault" +"toUInt32OrNull" +"toUInt32OrZero" +"toUInt64" +"toUInt64OrDefault" +"toUInt64OrNull" +"toUInt64OrZero" +"toUInt8" +"toUInt8OrDefault" +"toUInt8OrNull" +"toUInt8OrZero" +"toUTCTimestamp" +"toUUID" +"toUUIDOrDefault" +"toUUIDOrNull" +"toUUIDOrZero" +"toUnixTimestamp" +"toUnixTimestamp64Micro" +"toUnixTimestamp64Milli" +"toUnixTimestamp64Nano" +"toValidUTF8" +"toWeek" +"toYYYYMM" +"toYYYYMMDD" +"toYYYYMMDDhhmmss" +"toYear" +"toYearWeek" +"to_utc_timestamp" +"today" +"tokens" +"topK" +"topKArgMax" +"topKArgMin" +"topKArray" +"topKDistinct" +"topKForEach" +"topKIf" +"topKMap" +"topKMerge" +"topKNull" +"topKOrDefault" +"topKOrNull" +"topKResample" +"topKSimpleState" +"topKState" +"topKWeighted" +"topKWeightedArgMax" +"topKWeightedArgMin" +"topKWeightedArray" +"topKWeightedDistinct" +"topKWeightedForEach" +"topKWeightedIf" +"topKWeightedMap" +"topKWeightedMerge" +"topKWeightedNull" +"topKWeightedOrDefault" +"topKWeightedOrNull" +"topKWeightedResample" +"topKWeightedSimpleState" +"topKWeightedState" +"topLevelDomain" +"topLevelDomainRFC" +"transactionID" +"transactionLatestSnapshot" +"transactionOldestSnapshot" +"transform" +"translate" +"translateUTF8" +"trim" +"trimBoth" +"trimLeft" +"trimRight" +"trunc" +"truncate" +"tryBase58Decode" +"tryBase64Decode" +"tryDecrypt" +"tryIdnaEncode" +"tryPunycodeDecode" +"tumble" +"tumbleEnd" +"tumbleStart" +"tuple" +"tupleConcat" +"tupleDivide" +"tupleDivideByNumber" +"tupleElement" +"tupleHammingDistance" +"tupleIntDiv" +"tupleIntDivByNumber" +"tupleIntDivOrZero" +"tupleIntDivOrZeroByNumber" +"tupleMinus" +"tupleModulo" +"tupleModuloByNumber" +"tupleMultiply" +"tupleMultiplyByNumber" +"tupleNegate" +"tuplePlus" +"tupleToNameValuePairs" +"ucase" +"unbin" +"unhex" +"uniq" +"uniqArgMax" +"uniqArgMin" +"uniqArray" +"uniqCombined" +"uniqCombined64" +"uniqCombined64ArgMax" +"uniqCombined64ArgMin" +"uniqCombined64Array" +"uniqCombined64Distinct" +"uniqCombined64ForEach" +"uniqCombined64If" +"uniqCombined64Map" +"uniqCombined64Merge" +"uniqCombined64Null" +"uniqCombined64OrDefault" +"uniqCombined64OrNull" +"uniqCombined64Resample" +"uniqCombined64SimpleState" +"uniqCombined64State" +"uniqCombinedArgMax" +"uniqCombinedArgMin" +"uniqCombinedArray" +"uniqCombinedDistinct" +"uniqCombinedForEach" +"uniqCombinedIf" +"uniqCombinedMap" +"uniqCombinedMerge" +"uniqCombinedNull" +"uniqCombinedOrDefault" +"uniqCombinedOrNull" +"uniqCombinedResample" +"uniqCombinedSimpleState" +"uniqCombinedState" +"uniqDistinct" +"uniqExact" +"uniqExactArgMax" +"uniqExactArgMin" +"uniqExactArray" +"uniqExactDistinct" +"uniqExactForEach" +"uniqExactIf" +"uniqExactMap" +"uniqExactMerge" +"uniqExactNull" +"uniqExactOrDefault" +"uniqExactOrNull" +"uniqExactResample" +"uniqExactSimpleState" +"uniqExactState" +"uniqForEach" +"uniqHLL12" +"uniqHLL12ArgMax" +"uniqHLL12ArgMin" +"uniqHLL12Array" +"uniqHLL12Distinct" +"uniqHLL12ForEach" +"uniqHLL12If" +"uniqHLL12Map" +"uniqHLL12Merge" +"uniqHLL12Null" +"uniqHLL12OrDefault" +"uniqHLL12OrNull" +"uniqHLL12Resample" +"uniqHLL12SimpleState" +"uniqHLL12State" +"uniqIf" +"uniqMap" +"uniqMerge" +"uniqNull" +"uniqOrDefault" +"uniqOrNull" +"uniqResample" +"uniqSimpleState" +"uniqState" +"uniqTheta" +"uniqThetaArgMax" +"uniqThetaArgMin" +"uniqThetaArray" +"uniqThetaDistinct" +"uniqThetaForEach" +"uniqThetaIf" +"uniqThetaIntersect" +"uniqThetaMap" +"uniqThetaMerge" +"uniqThetaNot" +"uniqThetaNull" +"uniqThetaOrDefault" +"uniqThetaOrNull" +"uniqThetaResample" +"uniqThetaSimpleState" +"uniqThetaState" +"uniqThetaUnion" +"uniqUpTo" +"uniqUpToArgMax" +"uniqUpToArgMin" +"uniqUpToArray" +"uniqUpToDistinct" +"uniqUpToForEach" +"uniqUpToIf" +"uniqUpToMap" +"uniqUpToMerge" +"uniqUpToNull" +"uniqUpToOrDefault" +"uniqUpToOrNull" +"uniqUpToResample" +"uniqUpToSimpleState" +"uniqUpToState" +"upper" +"upperUTF8" +"uptime" +"user" +"validateNestedArraySizes" +"varPop" +"varPopArgMax" +"varPopArgMin" +"varPopArray" +"varPopDistinct" +"varPopForEach" +"varPopIf" +"varPopMap" +"varPopMerge" +"varPopNull" +"varPopOrDefault" +"varPopOrNull" +"varPopResample" +"varPopSimpleState" +"varPopStable" +"varPopStableArgMax" +"varPopStableArgMin" +"varPopStableArray" +"varPopStableDistinct" +"varPopStableForEach" +"varPopStableIf" +"varPopStableMap" +"varPopStableMerge" +"varPopStableNull" +"varPopStableOrDefault" +"varPopStableOrNull" +"varPopStableResample" +"varPopStableSimpleState" +"varPopStableState" +"varPopState" +"varSamp" +"varSampArgMax" +"varSampArgMin" +"varSampArray" +"varSampDistinct" +"varSampForEach" +"varSampIf" +"varSampMap" +"varSampMerge" +"varSampNull" +"varSampOrDefault" +"varSampOrNull" +"varSampResample" +"varSampSimpleState" +"varSampStable" +"varSampStableArgMax" +"varSampStableArgMin" +"varSampStableArray" +"varSampStableDistinct" +"varSampStableForEach" +"varSampStableIf" +"varSampStableMap" +"varSampStableMerge" +"varSampStableNull" +"varSampStableOrDefault" +"varSampStableOrNull" +"varSampStableResample" +"varSampStableSimpleState" +"varSampStableState" +"varSampState" +"variantElement" +"variantType" +"vectorDifference" +"vectorSum" +"version" +"visibleWidth" +"visitParamExtractBool" +"visitParamExtractFloat" +"visitParamExtractInt" +"visitParamExtractRaw" +"visitParamExtractString" +"visitParamExtractUInt" +"visitParamHas" +"week" +"welchTTest" +"welchTTestArgMax" +"welchTTestArgMin" +"welchTTestArray" +"welchTTestDistinct" +"welchTTestForEach" +"welchTTestIf" +"welchTTestMap" +"welchTTestMerge" +"welchTTestNull" +"welchTTestOrDefault" +"welchTTestOrNull" +"welchTTestResample" +"welchTTestSimpleState" +"welchTTestState" +"widthBucket" +"width_bucket" +"windowFunnel" +"windowFunnelArgMax" +"windowFunnelArgMin" +"windowFunnelArray" +"windowFunnelDistinct" +"windowFunnelForEach" +"windowFunnelIf" +"windowFunnelMap" +"windowFunnelMerge" +"windowFunnelNull" +"windowFunnelOrDefault" +"windowFunnelOrNull" +"windowFunnelResample" +"windowFunnelSimpleState" +"windowFunnelState" +"windowID" +"wkt" +"wordShingleMinHash" +"wordShingleMinHashArg" +"wordShingleMinHashArgCaseInsensitive" +"wordShingleMinHashArgCaseInsensitiveUTF8" +"wordShingleMinHashArgUTF8" +"wordShingleMinHashCaseInsensitive" +"wordShingleMinHashCaseInsensitiveUTF8" +"wordShingleMinHashUTF8" +"wordShingleSimHash" +"wordShingleSimHashCaseInsensitive" +"wordShingleSimHashCaseInsensitiveUTF8" +"wordShingleSimHashUTF8" +"wyHash64" +"xor" +"xxHash32" +"xxHash64" +"xxh3" +"yandexConsistentHash" +"yearweek" +"yesterday" +"zookeeperSessionUptime" diff --git a/tests/fuzz/dictionaries/key_words.dict b/tests/fuzz/dictionaries/key_words.dict index 0db2d159759..db517a2382c 100644 --- a/tests/fuzz/dictionaries/key_words.dict +++ b/tests/fuzz/dictionaries/key_words.dict @@ -1,183 +1,543 @@ -# Key words (based on keywords from antlr parser) - "ADD" +"ADD COLUMN" +"ADD CONSTRAINT" +"ADD INDEX" +"ADD PROJECTION" +"ADD STATISTIC" +"ADMIN OPTION FOR" "AFTER" +"ALGORITHM" "ALIAS" "ALL" +"ALLOWED_LATENESS" "ALTER" +"ALTER COLUMN" +"ALTER DATABASE" +"ALTER LIVE VIEW" +"ALTER POLICY" +"ALTER PROFILE" +"ALTER QUOTA" +"ALTER ROLE" +"ALTER ROW POLICY" +"ALTER SETTINGS PROFILE" +"ALTER TABLE" +"ALTER TEMPORARY TABLE" +"ALTER USER" "AND" +"AND STDOUT" "ANTI" "ANY" -"ARRAY" +"APPEND" +"APPLY" +"APPLY DELETED MASK" +"ARRAY JOIN" "AS" +"ASC" "ASCENDING" "ASOF" +"ASSUME" "AST" "ASYNC" "ATTACH" +"ATTACH PART" +"ATTACH PARTITION" +"ATTACH POLICY" +"ATTACH PROFILE" +"ATTACH QUOTA" +"ATTACH ROLE" +"ATTACH ROW POLICY" +"ATTACH SETTINGS PROFILE" +"ATTACH USER" +"AUTO_INCREMENT" +"AZURE" +"BACKUP" +"BCRYPT_HASH" +"BCRYPT_PASSWORD" +"BEGIN TRANSACTION" "BETWEEN" +"BIDIRECTIONAL" "BOTH" "BY" +"CASCADE" "CASE" "CAST" +"CHANGE" +"CHANGEABLE_IN_READONLY" +"CHANGED" +"CHAR" +"CHAR VARYING" +"CHARACTER" +"CHARACTER LARGE OBJECT" +"CHARACTER VARYING" "CHECK" -"CLEAR" +"CHECK ALL TABLES" +"CHECK TABLE" +"CLEANUP" +"CLEAR COLUMN" +"CLEAR INDEX" +"CLEAR PROJECTION" +"CLEAR STATISTIC" "CLUSTER" +"CLUSTERS" +"CN" "CODEC" "COLLATE" "COLUMN" +"COLUMNS" "COMMENT" +"COMMENT COLUMN" +"COMMIT" +"COMPRESSION" +"CONST" "CONSTRAINT" "CREATE" +"CREATE POLICY" +"CREATE PROFILE" +"CREATE QUOTA" +"CREATE ROLE" +"CREATE ROW POLICY" +"CREATE SETTINGS PROFILE" +"CREATE TABLE" +"CREATE TEMPORARY TABLE" +"CREATE USER" "CROSS" "CUBE" +"CURRENT GRANTS" +"CURRENT QUOTA" +"CURRENT ROLES" +"CURRENT ROW" +"CURRENT TRANSACTION" +"CURRENTUSER" +"CURRENT_USER" +"D" "DATABASE" "DATABASES" "DATE" "DAY" +"DAYS" +"DD" "DEDUPLICATE" "DEFAULT" -"DELAY" +"DEFAULT DATABASE" +"DEFAULT ROLE" +"DEFINER" "DELETE" +"DEPENDS ON" "DESC" "DESCENDING" "DESCRIBE" "DETACH" +"DETACH PART" +"DETACH PARTITION" "DICTIONARIES" "DICTIONARY" "DISK" "DISTINCT" -"DISTRIBUTED" +"DISTINCT ON" +"DIV" +"DOUBLE_SHA1_HASH" +"DOUBLE_SHA1_PASSWORD" "DROP" +"DROP COLUMN" +"DROP CONSTRAINT" +"DROP DEFAULT" +"DROP DETACHED PART" +"DROP DETACHED PARTITION" +"DROP INDEX" +"DROP PART" +"DROP PARTITION" +"DROP PROJECTION" +"DROP STATISTIC" +"DROP TABLE" +"DROP TEMPORARY TABLE" "ELSE" +"EMPTY" +"EMPTY AS" +"ENABLED ROLES" "END" +"ENFORCED" "ENGINE" +"EPHEMERAL" +"EPHEMERAL SEQUENTIAL" +"ESTIMATE" +"EVENT" "EVENTS" +"EVERY" +"EXCEPT" +"EXCEPT DATABASE" +"EXCEPT DATABASES" +"EXCEPT TABLE" +"EXCEPT TABLES" +"EXCHANGE DICTIONARIES" +"EXCHANGE TABLES" "EXISTS" "EXPLAIN" "EXPRESSION" -"EXTRACT" -"FETCHES" +"EXTENDED" +"EXTERNAL DDL FROM" +"FALSE" +"FETCH" +"FETCH PART" +"FETCH PARTITION" +"FIELDS" +"FILE" +"FILESYSTEM CACHE" +"FILESYSTEM CACHES" +"FILTER" "FINAL" "FIRST" -"FLUSH" +"FOLLOWING" "FOR" +"FOREIGN" +"FOREIGN KEY" +"FORGET PARTITION" "FORMAT" "FREEZE" "FROM" +"FROM INFILE" +"FROM SHARD" "FULL" +"FULLTEXT" "FUNCTION" "GLOBAL" +"GLOBAL IN" +"GLOBAL NOT IN" +"GRANT" +"GRANT OPTION FOR" +"GRANTEES" "GRANULARITY" -"GROUP" +"GROUP BY" +"GROUPING SETS" +"GROUPS" +"H" +"HASH" "HAVING" +"HDFS" +"HH" "HIERARCHICAL" +"HOST" "HOUR" +"HOURS" +"HTTP" "ID" -"IF" +"IDENTIFIED" +"IF EMPTY" +"IF EXISTS" +"IF NOT EXISTS" +"IGNORE NULLS" "ILIKE" "IN" +"IN PARTITION" "INDEX" -"INF" +"INDEXES" +"INDICES" +"INHERIT" "INJECTIVE" "INNER" -"INSERT" +"INSERT INTO" +"INTERPOLATE" +"INTERSECT" "INTERVAL" -"INTO" -"IS" +"INTO OUTFILE" +"INVISIBLE" +"INVOKER" +"IP" +"IS NOT DISTINCT FROM" +"IS NOT NULL" +"IS NULL" "IS_OBJECT_ID" "JOIN" +"KERBEROS" "KEY" +"KEY BY" +"KEYED BY" +"KEYS" "KILL" +"KIND" +"LARGE OBJECT" "LAST" "LAYOUT" +"LDAP" "LEADING" "LEFT" +"LEFT ARRAY JOIN" +"LESS THAN" +"LEVEL" "LIFETIME" +"LIGHTWEIGHT" "LIKE" "LIMIT" +"LINEAR" +"LIST" "LIVE" "LOCAL" -"LOGS" +"M" +"MATCH" "MATERIALIZE" +"MATERIALIZE COLUMN" +"MATERIALIZE INDEX" +"MATERIALIZE PROJECTION" +"MATERIALIZE STATISTIC" +"MATERIALIZE TTL" "MATERIALIZED" "MAX" +"MCS" +"MEMORY" "MERGES" +"MI" +"MICROSECOND" +"MICROSECONDS" +"MILLISECOND" +"MILLISECONDS" "MIN" "MINUTE" +"MINUTES" +"MM" +"MOD" "MODIFY" +"MODIFY COLUMN" +"MODIFY COMMENT" +"MODIFY ORDER BY" +"MODIFY QUERY" +"MODIFY REFRESH" +"MODIFY SAMPLE BY" +"MODIFY SETTING" +"MODIFY SQL SECURITY" +"MODIFY TTL" "MONTH" +"MONTHS" "MOVE" +"MOVE PART" +"MOVE PARTITION" +"MS" "MUTATION" -"NAN_SQL" -"NO" +"N" +"NAME" +"NAMED COLLECTION" +"NANOSECOND" +"NANOSECONDS" +"NEXT" +"NO ACTION" +"NO DELAY" +"NO LIMITS" +"NONE" "NOT" -"NULL_SQL" +"NOT BETWEEN" +"NOT IDENTIFIED" +"NOT ILIKE" +"NOT IN" +"NOT KEYED" +"NOT LIKE" +"NOT OVERRIDABLE" +"NO_PASSWORD" +"NS" +"NULL" "NULLS" "OFFSET" "ON" -"OPTIMIZE" +"ON DELETE" +"ON UPDATE" +"ON VOLUME" +"ONLY" +"OPTIMIZE TABLE" "OR" -"ORDER" +"OR REPLACE" +"ORDER BY" "OUTER" -"OUTFILE" +"OVER" +"OVERRIDABLE" +"PART" +"PARTIAL" "PARTITION" +"PARTITION BY" +"PARTITIONS" +"PART_MOVE_TO_SHARD" +"PASTE" +"PERIODIC REFRESH" +"PERMANENTLY" +"PERMISSIVE" +"PERSISTENT" +"PERSISTENT SEQUENTIAL" +"PIPELINE" +"PLAINTEXT_PASSWORD" +"PLAN" "POPULATE" +"PRECEDING" +"PRECISION" "PREWHERE" "PRIMARY" +"PRIMARY KEY" +"PROFILE" "PROJECTION" +"PULL" +"Protobuf" +"Q" +"QQ" "QUARTER" +"QUARTERS" +"QUERY" +"QUERY TREE" +"QUOTA" +"RANDOMIZE FOR" +"RANDOMIZED" "RANGE" -"RELOAD" +"READONLY" +"REALM" +"RECOMPRESS" +"REFERENCES" +"REFRESH" +"REGEXP" "REMOVE" +"REMOVE SAMPLE BY" +"REMOVE TTL" "RENAME" +"RENAME COLUMN" +"RENAME DATABASE" +"RENAME DICTIONARY" +"RENAME TABLE" +"RENAME TO" "REPLACE" -"REPLICA" -"REPLICATED" +"REPLACE PARTITION" +"RESET SETTING" +"RESPECT NULLS" +"RESTORE" +"RESTRICT" +"RESTRICTIVE" +"RESUME" +"REVOKE" "RIGHT" +"ROLLBACK" "ROLLUP" +"ROW" +"ROWS" +"S" +"S3" +"SALT" "SAMPLE" +"SAMPLE BY" +"SCHEME" "SECOND" +"SECONDS" "SELECT" "SEMI" -"SENDS" +"SERVER" "SET" +"SET DEFAULT" +"SET DEFAULT ROLE" +"SET FAKE TIME" +"SET NULL" +"SET ROLE" +"SET ROLE DEFAULT" +"SET TRANSACTION SNAPSHOT" "SETTINGS" +"SHA256_HASH" +"SHA256_PASSWORD" "SHOW" +"SHOW ACCESS" +"SHOW CREATE" +"SHOW ENGINES" +"SHOW FUNCTIONS" +"SHOW GRANTS" +"SHOW PRIVILEGES" +"SHOW PROCESSLIST" +"SHOW SETTING" +"SIGNED" +"SIMPLE" "SOURCE" -"START" -"STOP" -"SUBSTRING" +"SPATIAL" +"SQL SECURITY" +"SQL_TSI_DAY" +"SQL_TSI_HOUR" +"SQL_TSI_MICROSECOND" +"SQL_TSI_MILLISECOND" +"SQL_TSI_MINUTE" +"SQL_TSI_MONTH" +"SQL_TSI_NANOSECOND" +"SQL_TSI_QUARTER" +"SQL_TSI_SECOND" +"SQL_TSI_WEEK" +"SQL_TSI_YEAR" +"SS" +"SSH_KEY" +"SSL_CERTIFICATE" +"STATISTIC" +"STEP" +"STORAGE" +"STRICT" +"STRICTLY_ASCENDING" +"SUBPARTITION" +"SUBPARTITION BY" +"SUBPARTITIONS" +"SUSPEND" "SYNC" "SYNTAX" "SYSTEM" "TABLE" +"TABLE OVERRIDE" "TABLES" "TEMPORARY" +"TEMPORARY TABLE" "TEST" "THEN" -"TIES" -"TIMEOUT" "TIMESTAMP" "TO" +"TO DISK" +"TO INNER UUID" +"TO SHARD" +"TO TABLE" +"TO VOLUME" "TOP" "TOTALS" +"TRACKING ONLY" "TRAILING" -"TRIM" +"TRANSACTION" +"TRIGGER" +"TRUE" "TRUNCATE" "TTL" "TYPE" +"TYPEOF" +"UNBOUNDED" +"UNDROP" +"UNFREEZE" "UNION" +"UNIQUE" +"UNSET FAKE TIME" +"UNSIGNED" "UPDATE" +"URL" "USE" "USING" "UUID" +"VALID UNTIL" "VALUES" +"VARYING" "VIEW" -"VOLUME" +"VISIBLE" "WATCH" +"WATERMARK" "WEEK" +"WEEKS" "WHEN" "WHERE" +"WINDOW" "WITH" +"WITH ADMIN OPTION" +"WITH CHECK" +"WITH FILL" +"WITH GRANT OPTION" +"WITH NAME" +"WITH REPLACE OPTION" +"WITH TIES" +"WK" +"WRITABLE" +"WW" "YEAR" +"YEARS" +"YY" +"YYYY" +"ZKPATH" +"bagexpansion" +"base_backup" +"cluster_host_ids" +"with_itemindex" From a55fabdd5d6c46ab677f081bb4ec8a7491771ca4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:10:22 +0100 Subject: [PATCH 233/332] Inhibit randomization in the test --- tests/queries/0_stateless/02998_primary_key_skip_columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02998_primary_key_skip_columns.sql b/tests/queries/0_stateless/02998_primary_key_skip_columns.sql index b567155ab1f..6574a82a165 100644 --- a/tests/queries/0_stateless/02998_primary_key_skip_columns.sql +++ b/tests/queries/0_stateless/02998_primary_key_skip_columns.sql @@ -1,4 +1,4 @@ --- Tags: no-asan, no-tsan, no-msan, no-ubsan +-- Tags: no-asan, no-tsan, no-msan, no-ubsan, no-random-settings DROP TABLE IF EXISTS test; From 15aff51dfcae4679238c7cf78e542e40a2f4d4c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:13:26 +0100 Subject: [PATCH 234/332] Inhibit randomization in the test --- .../queries/0_stateless/01019_alter_materialized_view_atomic.sh | 2 +- .../0_stateless/01171_mv_select_insert_isolation_long.sh | 2 +- .../0_stateless/01459_manual_write_to_replicas_quorum.sh | 2 +- tests/queries/0_stateless/01508_partition_pruning_long.sh | 2 +- tests/queries/0_stateless/01747_system_session_log_long.sh | 2 +- .../0_stateless/02782_uniq_exact_parallel_merging_bug.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index e50b974a48e..4bd21fcee02 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-debug set -e diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh index 619e212c3ae..8344bb6f426 100755 --- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh +++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel, no-ordinary-database +# Tags: long, no-parallel, no-ordinary-database, no-debug # Test is too heavy, avoid parallel run in Flaky Check # shellcheck disable=SC2119 diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index 379f83c6271..91a73471557 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: replica, no-replicated-database, no-parallel +# Tags: replica, no-replicated-database, no-parallel, no-debug, no-random-settings # Tag no-replicated-database: Fails due to additional replicas or shards set -e diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh index c1f2d6562ab..8e4b9547da8 100755 --- a/tests/queries/0_stateless/01508_partition_pruning_long.sh +++ b/tests/queries/0_stateless/01508_partition_pruning_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings +# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings, no-debug # Description of test result: # Test the correctness of the partition pruning diff --git a/tests/queries/0_stateless/01747_system_session_log_long.sh b/tests/queries/0_stateless/01747_system_session_log_long.sh index 3ad33c28ee1..ecddcb627b8 100755 --- a/tests/queries/0_stateless/01747_system_session_log_long.sh +++ b/tests/queries/0_stateless/01747_system_session_log_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel, no-fasttest +# Tags: long, no-parallel, no-fasttest, no-debug ################################################################################################## # Verify that login, logout, and login failure events are properly stored in system.session_log diff --git a/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh b/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh index 060efe86602..6e787523540 100755 --- a/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh +++ b/tests/queries/0_stateless/02782_uniq_exact_parallel_merging_bug.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-random-settings, no-tsan, no-asan, no-ubsan, no-msan, no-parallel +# Tags: long, no-random-settings, no-tsan, no-asan, no-ubsan, no-msan, no-parallel, no-debug # shellcheck disable=SC2154 From 618c11166d5ada0b2b37a69eef534b93997faaf9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 00:53:09 +0100 Subject: [PATCH 235/332] Add a test for #61669 --- .../0_stateless/03015_peder1001.reference | 1 + tests/queries/0_stateless/03015_peder1001.sql | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/03015_peder1001.reference create mode 100644 tests/queries/0_stateless/03015_peder1001.sql diff --git a/tests/queries/0_stateless/03015_peder1001.reference b/tests/queries/0_stateless/03015_peder1001.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03015_peder1001.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03015_peder1001.sql b/tests/queries/0_stateless/03015_peder1001.sql new file mode 100644 index 00000000000..810503207f2 --- /dev/null +++ b/tests/queries/0_stateless/03015_peder1001.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS test_data; + +CREATE TABLE test_data +( + ShipmentDate Date +) +ENGINE = Memory; + +INSERT INTO test_data (ShipmentDate) Values ('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'), ('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-02-07'), ('2022-02-11'), ('2022-02-15'), ('2022-02-16'), ('2022-02-18'), ('2022-02-22'), ('2022-02-24'), ('2022-03-02'), ('2022-03-04'), ('2022-03-07'), ('2022-03-09'), ('2022-03-10'), ('2022-03-11'), ('2022-03-14'), ('2022-03-15'), ('2022-03-17'), ('2022-03-18'), ('2022-03-23'), ('2022-04-28'), ('2022-05-24'), ('2022-03-31'), ('2022-04-19'), ('2022-04-25'), ('2022-04-26'), ('2022-05-02'), ('2022-05-04'), ('2022-05-05'), ('2022-05-11'), ('2022-05-12'), ('2022-05-13'), ('2022-05-16'), ('2022-05-18'), ('2022-05-20'), ('2022-05-23'), ('2022-05-27'), ('2022-05-31'), ('2022-05-10'), ('2022-02-17'), ('2022-03-24'), ('2022-05-09'), ('2022-05-30'), ('2022-02-21'), ('2022-01-11'), ('2022-01-28'), ('2022-04-27'), ('2022-05-25'), ('2022-04-18'), ('2022-01-21'), ('2022-03-22'), ('2022-04-01'), ('2022-04-06'), ('2022-04-11'), ('2022-05-19'), ('2022-02-01'), ('2022-02-23'), ('2022-02-09'), ('2022-03-03'), ('2022-04-04'), ('2022-04-05'), ('2022-04-12'), ('2022-04-29'), ('2022-01-06'), ('2022-03-01'), ('2022-03-26'), ('2022-01-10'), ('2022-01-03'), ('2022-05-01'), ('2022-03-21'), ('2022-03-27'), ('2022-01-31'), ('2022-04-13'), ('2022-03-29'), ('2022-02-20'), ('2022-02-06'), ('2022-03-13'), ('2022-02-27'), ('2022-03-20'), ('2022-04-24'), ('2022-05-15'), ('2022-05-22'), ('2022-01-09'), ('2022-04-03'), ('2022-03-12'), ('2022-01-23'), ('2022-05-08'), ('2022-05-29'), ('2022-02-19'), ('2022-05-07'), ('2022-05-26'), ('2022-01-30'), ('2022-03-05'), ('2022-05-21'), ('2022-02-26'), ('2022-01-16'), ('2022-05-17'), ('2022-01-29'), ('2022-02-12'), ('2022-01-02'), ('2022-02-05'),('2022-04-22'), ('2022-02-14'), ('2022-02-28'), ('2022-02-04'), ('2022-02-08'), ('2022-03-16'), ('2022-03-25'), ('2022-02-25'), ('2022-03-08'), ('2022-05-03'), ('2022-05-06'), ('2022-02-10'), ('2022-02-13'), ('2022-03-06'), ('2022-04-07'), ('2022-04-08'), ('2022-04-20'), ('2022-04-21'), ('2022-03-28'), ('2022-03-30'), ('2022-01-04'), ('2022-01-05'), ('2022-01-07'), ('2022-01-12'), ('2022-01-13'), ('2022-01-14'), ('2022-01-17'), ('2022-01-18'), ('2022-01-19'), ('2022-01-20'), ('2022-01-24'), ('2022-01-25'), ('2022-01-26'), ('2022-01-27'), ('2022-02-02'), ('2022-02-03'), ('2022-01-08'); + +SELECT + toDayOfWeek(ShipmentDate) AS c +FROM test_data +WHERE c IS NOT NULL AND lowerUTF8(formatDateTime(date_add(DAY, toInt32(c) - 1, toDate('2024-01-01')), '%W')) LIKE '%m%' +GROUP BY c +ORDER BY c ASC +LIMIT 62 +OFFSET 0; + +DROP TABLE test_data; From 93dd212f6f9602e690ec3f378691ae57b29d1112 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Wed, 20 Mar 2024 23:55:25 +0000 Subject: [PATCH 236/332] Fix use-of-uninitialized-value in HedgedConnections --- src/Client/HedgedConnections.cpp | 1 - src/Client/HedgedConnections.h | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 9fa79dd6b77..fb4d9a6bdcc 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -67,7 +67,6 @@ HedgedConnections::HedgedConnections( } active_connection_count = connections.size(); - offsets_with_disabled_changing_replica = 0; pipeline_for_new_replicas.add([throttler_](ReplicaState & replica_) { replica_.connection->setThrottler(throttler_); }); } diff --git a/src/Client/HedgedConnections.h b/src/Client/HedgedConnections.h index 5bc274332db..7f538804e5a 100644 --- a/src/Client/HedgedConnections.h +++ b/src/Client/HedgedConnections.h @@ -178,12 +178,12 @@ private: std::queue offsets_queue; /// The current number of valid connections to the replicas of this shard. - size_t active_connection_count; + size_t active_connection_count = 0; /// We count offsets in which we can't change replica anymore, /// it's needed to cancel choosing new replicas when we /// disabled replica changing in all offsets. - size_t offsets_with_disabled_changing_replica; + size_t offsets_with_disabled_changing_replica = 0; Pipeline pipeline_for_new_replicas; From 4d6709579bef2bf23ba8d8cd9ee6e925dd257836 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:28:14 +0100 Subject: [PATCH 237/332] Fix error --- src/Analyzer/MatcherNode.cpp | 8 ++++---- src/Analyzer/MatcherNode.h | 2 +- src/Parsers/ASTColumnsTransformers.cpp | 23 +++++++++++++---------- src/Parsers/ASTColumnsTransformers.h | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 48b60423688..f573b83e538 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -92,19 +92,19 @@ MatcherNode::MatcherNode(Identifier qualified_identifier_, Identifiers columns_i MatcherNode::MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - String pattern_, + std::optional pattern_, ColumnTransformersNodes column_transformers_) : IQueryTreeNode(children_size) , matcher_type(matcher_type_) , qualified_identifier(qualified_identifier_) , columns_identifiers(columns_identifiers_) { - if (!pattern_.empty()) + if (pattern_) { - columns_matcher = std::make_shared(pattern_, re2::RE2::Quiet); + columns_matcher = std::make_shared(*pattern_, re2::RE2::Quiet); if (!columns_matcher->ok()) throw DB::Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern_, columns_matcher->error()); + "COLUMNS pattern {} cannot be compiled: {}", *pattern_, columns_matcher->error()); } auto column_transformers_list_node = std::make_shared(); diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index 9f6b7c6ce3d..d6f077e224b 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -147,7 +147,7 @@ private: explicit MatcherNode(MatcherNodeType matcher_type_, Identifier qualified_identifier_, Identifiers columns_identifiers_, - String pattern_, + std::optional pattern_, ColumnTransformersNodes column_transformers_); MatcherNodeType matcher_type; diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 42c4c6ad88a..2a61892f8cc 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -180,8 +180,8 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo (*it)->formatImpl(settings, state, frame); } - if (!pattern.empty()) - settings.ostr << quoteString(pattern); + if (pattern) + settings.ostr << quoteString(*pattern); if (children.size() > 1) settings.ostr << ")"; @@ -203,8 +203,8 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const (*it)->appendColumnName(ostr); } - if (!pattern.empty()) - writeQuotedString(pattern, ostr); + if (pattern) + writeQuotedString(*pattern, ostr); if (children.size() > 1) writeChar(')', ostr); @@ -213,8 +213,11 @@ void ASTColumnsExceptTransformer::appendColumnName(WriteBuffer & ostr) const void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const { hash_state.update(is_strict); - hash_state.update(pattern.size()); - hash_state.update(pattern); + if (pattern) + { + hash_state.update(pattern->size()); + hash_state.update(*pattern); + } IAST::updateTreeHashImpl(hash_state, ignore_aliases); } @@ -222,7 +225,7 @@ void ASTColumnsExceptTransformer::updateTreeHashImpl(SipHash & hash_state, bool void ASTColumnsExceptTransformer::transform(ASTs & nodes) const { std::set expected_columns; - if (pattern.empty()) + if (!pattern) { for (const auto & child : children) expected_columns.insert(child->as().name()); @@ -278,13 +281,13 @@ void ASTColumnsExceptTransformer::setPattern(String pattern_) std::shared_ptr ASTColumnsExceptTransformer::getMatcher() const { - if (pattern.empty()) + if (!pattern) return {}; - auto regexp = std::make_shared(pattern, re2::RE2::Quiet); + auto regexp = std::make_shared(*pattern, re2::RE2::Quiet); if (!regexp->ok()) throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP, - "COLUMNS pattern {} cannot be compiled: {}", pattern, regexp->error()); + "COLUMNS pattern {} cannot be compiled: {}", *pattern, regexp->error()); return regexp; } diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index 2d13cd85819..cedf955bee2 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -84,7 +84,7 @@ public: protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - String pattern; + std::optional pattern; }; class ASTColumnsReplaceTransformer : public IASTColumnsTransformer From 3698c0c2cfb0cd5cda2c10531ca6dcfc1e9899e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:46:03 +0100 Subject: [PATCH 238/332] Fix build --- src/Functions/FunctionsTimeWindow.cpp | 11 +++++ src/Functions/FunctionsTimeWindow.h | 62 +++------------------------ 2 files changed, 17 insertions(+), 56 deletions(-) diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index be51978499b..2b1359964e8 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -110,6 +110,17 @@ bool checkIntervalOrTimeZoneArgument(const ColumnWithTypeAndName & argument, con return true; } +enum TimeWindowFunctionName +{ + TUMBLE, + TUMBLE_START, + TUMBLE_END, + HOP, + HOP_START, + HOP_END, + WINDOW_ID +}; + template struct TimeWindowImpl { diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 4ea5b3a9394..65cc7e9e87c 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -21,16 +21,6 @@ namespace DB * hopEnd(window_id) * hopEnd(time_attr, hop_interval, window_interval [, timezone]) */ -enum TimeWindowFunctionName -{ - TUMBLE, - TUMBLE_START, - TUMBLE_END, - HOP, - HOP_START, - HOP_END, - WINDOW_ID -}; template struct ToStartOfTransform; @@ -73,17 +63,15 @@ struct ToStartOfTransform; TRANSFORM_TIME(Second) #undef TRANSFORM_TIME -/// NOLINTBEGIN(bugprone-macro-parentheses) - #define TRANSFORM_SUBSECONDS(INTERVAL_KIND, DEF_SCALE) \ template<> \ struct ToStartOfTransform \ { \ static Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ { \ - if (scale <= DEF_SCALE) \ + if (scale <= (DEF_SCALE)) \ { \ - auto val = t * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + auto val = t * DecimalUtils::scaleMultiplier((DEF_SCALE) - scale); \ if (delta == 1) \ return val; \ else \ @@ -91,7 +79,7 @@ template<> \ } \ else \ { \ - return t - (t % (delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE))) ; \ + return t - (t % (delta * DecimalUtils::scaleMultiplier(scale - (DEF_SCALE)))) ; \ } \ } \ }; @@ -145,12 +133,12 @@ template <> \ { \ static inline NO_SANITIZE_UNDEFINED Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ { \ - if (scale < DEF_SCALE) \ + if (scale < (DEF_SCALE)) \ { \ - return t + delta * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + return t + delta * DecimalUtils::scaleMultiplier((DEF_SCALE) - scale); \ } \ else \ - return t + delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE); \ + return t + delta * DecimalUtils::scaleMultiplier(scale - (DEF_SCALE)); \ } \ }; ADD_SUBSECONDS(Millisecond, 3) @@ -158,42 +146,4 @@ template <> \ ADD_SUBSECONDS(Nanosecond, 9) #undef ADD_SUBSECONDS -/// NOLINTEND(bugprone-macro-parentheses) - -template -struct TimeWindowImpl -{ - static constexpr auto name = "UNKNOWN"; - - static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name); - - static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name); -}; - -template -class FunctionTimeWindow : public IFunction -{ -public: - static constexpr auto name = TimeWindowImpl::name; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } - bool isVariadic() const override { return true; } - size_t getNumberOfArguments() const override { return 0; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2, 3}; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override; -}; - -using FunctionTumble = FunctionTimeWindow; -using FunctionTumbleStart = FunctionTimeWindow; -using FunctionTumbleEnd = FunctionTimeWindow; -using FunctionHop = FunctionTimeWindow; -using FunctionWindowId = FunctionTimeWindow; -using FunctionHopStart = FunctionTimeWindow; -using FunctionHopEnd = FunctionTimeWindow; - } From bfc7c3c89edd2454ee0d6d23052d9bade28ac2a7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 02:52:51 +0100 Subject: [PATCH 239/332] Remove clickhouse-diagnostics --- docker/packager/binary-builder/Dockerfile | 2 +- docker/packager/binary-builder/build.sh | 18 - packages/clickhouse-common-static.yaml | 2 - programs/diagnostics/.gitignore | 30 - programs/diagnostics/CONTRIBUTION.md | 49 - programs/diagnostics/Makefile | 65 - programs/diagnostics/README.md | 167 --- .../cmd/clickhouse-diagnostics/main.go | 9 - programs/diagnostics/cmd/collect.go | 159 --- programs/diagnostics/cmd/convert.go | 1 - programs/diagnostics/cmd/help.go | 124 -- programs/diagnostics/cmd/params/params.go | 281 ---- .../diagnostics/cmd/params/params_test.go | 247 ---- programs/diagnostics/cmd/root.go | 174 --- programs/diagnostics/cmd/version.go | 24 - programs/diagnostics/go.mod | 89 -- programs/diagnostics/go.sum | 992 -------------- .../internal/collectors/clickhouse/config.go | 113 -- .../collectors/clickhouse/config_test.go | 128 -- .../internal/collectors/clickhouse/db_logs.go | 108 -- .../collectors/clickhouse/db_logs_test.go | 119 -- .../internal/collectors/clickhouse/logs.go | 140 -- .../collectors/clickhouse/logs_test.go | 147 -- .../collectors/clickhouse/queries.json | 153 --- .../internal/collectors/clickhouse/summary.go | 159 --- .../collectors/clickhouse/summary_test.go | 111 -- .../internal/collectors/clickhouse/system.go | 165 --- .../collectors/clickhouse/system_test.go | 366 ----- .../collectors/clickhouse/zookeeper.go | 153 --- .../collectors/clickhouse/zookeeper_test.go | 102 -- .../internal/collectors/registry.go | 75 -- .../internal/collectors/registry_test.go | 57 - .../internal/collectors/system/command.go | 90 -- .../collectors/system/command_test.go | 107 -- .../internal/collectors/system/file.go | 100 -- .../internal/collectors/system/file_test.go | 110 -- .../internal/collectors/system/system.go | 235 ---- .../internal/collectors/system/system_test.go | 89 -- .../internal/outputs/file/simple.go | 344 ----- .../internal/outputs/file/simple_test.go | 468 ------- .../diagnostics/internal/outputs/registry.go | 67 - .../internal/outputs/registry_test.go | 45 - .../internal/outputs/terminal/report.go | 284 ---- .../internal/platform/config/models.go | 129 -- .../internal/platform/config/models_test.go | 182 --- .../internal/platform/config/utils.go | 74 - .../internal/platform/config/utils_test.go | 142 -- .../internal/platform/data/bundle.go | 27 - .../internal/platform/data/bundle_test.go | 26 - .../internal/platform/data/database.go | 88 -- .../internal/platform/data/database_test.go | 86 -- .../internal/platform/data/field.go | 8 - .../internal/platform/data/file.go | 444 ------ .../internal/platform/data/file_test.go | 263 ---- .../internal/platform/data/frame.go | 11 - .../internal/platform/data/memory.go | 35 - .../internal/platform/data/memory_test.go | 61 - .../internal/platform/data/misc.go | 27 - .../internal/platform/database/native.go | 95 -- .../internal/platform/database/native_test.go | 289 ---- .../diagnostics/internal/platform/manager.go | 49 - .../internal/platform/manager_test.go | 100 -- .../internal/platform/test/data.go | 166 --- .../diagnostics/internal/platform/test/env.go | 16 - .../internal/platform/utils/file.go | 95 -- .../internal/platform/utils/file_test.go | 134 -- .../internal/platform/utils/process.go | 49 - .../internal/platform/utils/process_test.go | 97 -- .../internal/platform/utils/slices.go | 68 - .../internal/platform/utils/slices_test.go | 64 - .../internal/platform/utils/time.go | 7 - programs/diagnostics/internal/runner.go | 115 -- programs/diagnostics/internal/runner_test.go | 130 -- .../configs/include/xml/server-include.xml | 8 - .../configs/include/xml/user-include.xml | 20 - .../configs/include/yaml/server-include.yaml | 1 - .../configs/include/yaml/user-include.yaml | 7 - .../testdata/configs/xml/config.xml | 1195 ----------------- .../configs/xml/users.d/default-password.xml | 8 - .../testdata/configs/xml/users.xml | 57 - .../testdata/configs/yaml/config.yaml | 927 ------------- .../yaml/users.d/default-password.yaml | 6 - .../testdata/configs/yaml/users.yaml | 47 - .../testdata/configs/yandex_xml/config.xml | 1167 ---------------- .../diagnostics/testdata/docker/admin.xml | 15 - .../diagnostics/testdata/docker/custom.xml | 8 - .../logs/var/logs/clickhouse-server.err.log | 10 - .../logs/var/logs/clickhouse-server.log | 10 - .../logs/var/logs/clickhouse-server.log.gz | 1 - 89 files changed, 1 insertion(+), 13031 deletions(-) delete mode 100644 programs/diagnostics/.gitignore delete mode 100644 programs/diagnostics/CONTRIBUTION.md delete mode 100644 programs/diagnostics/Makefile delete mode 100644 programs/diagnostics/README.md delete mode 100644 programs/diagnostics/cmd/clickhouse-diagnostics/main.go delete mode 100644 programs/diagnostics/cmd/collect.go delete mode 100644 programs/diagnostics/cmd/convert.go delete mode 100644 programs/diagnostics/cmd/help.go delete mode 100644 programs/diagnostics/cmd/params/params.go delete mode 100644 programs/diagnostics/cmd/params/params_test.go delete mode 100644 programs/diagnostics/cmd/root.go delete mode 100644 programs/diagnostics/cmd/version.go delete mode 100644 programs/diagnostics/go.mod delete mode 100644 programs/diagnostics/go.sum delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/config.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/config_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/db_logs.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/logs.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/logs_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/queries.json delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/summary.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/summary_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/system.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/system_test.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/zookeeper.go delete mode 100644 programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go delete mode 100644 programs/diagnostics/internal/collectors/registry.go delete mode 100644 programs/diagnostics/internal/collectors/registry_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/command.go delete mode 100644 programs/diagnostics/internal/collectors/system/command_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/file.go delete mode 100644 programs/diagnostics/internal/collectors/system/file_test.go delete mode 100644 programs/diagnostics/internal/collectors/system/system.go delete mode 100644 programs/diagnostics/internal/collectors/system/system_test.go delete mode 100644 programs/diagnostics/internal/outputs/file/simple.go delete mode 100644 programs/diagnostics/internal/outputs/file/simple_test.go delete mode 100644 programs/diagnostics/internal/outputs/registry.go delete mode 100644 programs/diagnostics/internal/outputs/registry_test.go delete mode 100644 programs/diagnostics/internal/outputs/terminal/report.go delete mode 100644 programs/diagnostics/internal/platform/config/models.go delete mode 100644 programs/diagnostics/internal/platform/config/models_test.go delete mode 100644 programs/diagnostics/internal/platform/config/utils.go delete mode 100644 programs/diagnostics/internal/platform/config/utils_test.go delete mode 100644 programs/diagnostics/internal/platform/data/bundle.go delete mode 100644 programs/diagnostics/internal/platform/data/bundle_test.go delete mode 100644 programs/diagnostics/internal/platform/data/database.go delete mode 100644 programs/diagnostics/internal/platform/data/database_test.go delete mode 100644 programs/diagnostics/internal/platform/data/field.go delete mode 100644 programs/diagnostics/internal/platform/data/file.go delete mode 100644 programs/diagnostics/internal/platform/data/file_test.go delete mode 100644 programs/diagnostics/internal/platform/data/frame.go delete mode 100644 programs/diagnostics/internal/platform/data/memory.go delete mode 100644 programs/diagnostics/internal/platform/data/memory_test.go delete mode 100644 programs/diagnostics/internal/platform/data/misc.go delete mode 100644 programs/diagnostics/internal/platform/database/native.go delete mode 100644 programs/diagnostics/internal/platform/database/native_test.go delete mode 100644 programs/diagnostics/internal/platform/manager.go delete mode 100644 programs/diagnostics/internal/platform/manager_test.go delete mode 100644 programs/diagnostics/internal/platform/test/data.go delete mode 100644 programs/diagnostics/internal/platform/test/env.go delete mode 100644 programs/diagnostics/internal/platform/utils/file.go delete mode 100644 programs/diagnostics/internal/platform/utils/file_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/process.go delete mode 100644 programs/diagnostics/internal/platform/utils/process_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/slices.go delete mode 100644 programs/diagnostics/internal/platform/utils/slices_test.go delete mode 100644 programs/diagnostics/internal/platform/utils/time.go delete mode 100644 programs/diagnostics/internal/runner.go delete mode 100644 programs/diagnostics/internal/runner_test.go delete mode 100644 programs/diagnostics/testdata/configs/include/xml/server-include.xml delete mode 100644 programs/diagnostics/testdata/configs/include/xml/user-include.xml delete mode 100644 programs/diagnostics/testdata/configs/include/yaml/server-include.yaml delete mode 100644 programs/diagnostics/testdata/configs/include/yaml/user-include.yaml delete mode 100644 programs/diagnostics/testdata/configs/xml/config.xml delete mode 100644 programs/diagnostics/testdata/configs/xml/users.d/default-password.xml delete mode 100644 programs/diagnostics/testdata/configs/xml/users.xml delete mode 100644 programs/diagnostics/testdata/configs/yaml/config.yaml delete mode 100644 programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml delete mode 100644 programs/diagnostics/testdata/configs/yaml/users.yaml delete mode 100644 programs/diagnostics/testdata/configs/yandex_xml/config.xml delete mode 100644 programs/diagnostics/testdata/docker/admin.xml delete mode 100644 programs/diagnostics/testdata/docker/custom.xml delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log delete mode 100644 programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index c9442accd7e..73ec4275f12 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -61,7 +61,7 @@ RUN arch=${TARGETARCH:-amd64} \ && rm /tmp/nfpm.deb ARG GO_VERSION=1.19.10 -# We need go for clickhouse-diagnostics +# We needed go for clickhouse-diagnostics (it is not used anymore) RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \ && tar -xzf /tmp/go.tgz -C /usr/local/ \ diff --git a/docker/packager/binary-builder/build.sh b/docker/packager/binary-builder/build.sh index b63643419fe..032aceb0af3 100755 --- a/docker/packager/binary-builder/build.sh +++ b/docker/packager/binary-builder/build.sh @@ -36,22 +36,6 @@ rm -f CMakeCache.txt if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root - # NOTE: this is for backward compatibility with previous releases, - # that does not diagnostics tool (only script). - if [ -d /build/programs/diagnostics ]; then - if [ -z "$SANITIZER" ]; then - # We need to check if clickhouse-diagnostics is fine and build it - ( - cd /build/programs/diagnostics - make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build - mv clickhouse-diagnostics .. - ) - else - echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics - chmod +x /build/programs/clickhouse-diagnostics - fi - fi fi @@ -121,8 +105,6 @@ if [ -n "$MAKE_DEB" ]; then # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 DESTDIR=/build/packages/root ninja $NINJA_FLAGS programs/install - cp /build/programs/clickhouse-diagnostics /build/packages/root/usr/bin - cp /build/programs/clickhouse-diagnostics /output bash -x /build/packages/build fi diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 238126f95fd..383ad39591c 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -34,8 +34,6 @@ suggests: contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse -- src: root/usr/bin/clickhouse-diagnostics - dst: /usr/bin/clickhouse-diagnostics - src: root/usr/bin/clickhouse-extract-from-config dst: /usr/bin/clickhouse-extract-from-config - src: root/usr/bin/clickhouse-library-bridge diff --git a/programs/diagnostics/.gitignore b/programs/diagnostics/.gitignore deleted file mode 100644 index 5e0b0165f38..00000000000 --- a/programs/diagnostics/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work - -.idea -clickhouse-diagnostics -output -vendor -bin -profile.cov -clickhouse-diagnostics.yml -dist/ diff --git a/programs/diagnostics/CONTRIBUTION.md b/programs/diagnostics/CONTRIBUTION.md deleted file mode 100644 index 00fb073cefe..00000000000 --- a/programs/diagnostics/CONTRIBUTION.md +++ /dev/null @@ -1,49 +0,0 @@ -# Contribution - -We keep things simple. Execute all commands in this folder. - -## Requirements - -- docker - tested on version 20.10.12. -- golang >= go1.17.6 - -## Building - -Creates a binary `clickhouse-diagnostics` in the local folder. Build will be versioned according to a timestamp. For a versioned release see [Releasing](#releasing). - -```bash -make build -``` - -## Linting - -We use [golangci-lint](https://golangci-lint.run/). We use a container to run so no need to install. - -```bash -make lint-go -``` - -## Running Tests - -```bash -make test -``` - -For a coverage report, - -```bash -make test-coverage -``` - -## Adding Collectors - -TODO - - -## Adding Outputs - -TODO - -## Frames - -## Parameter Types diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile deleted file mode 100644 index 2e85002b871..00000000000 --- a/programs/diagnostics/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -GOCMD=go -GOTEST=$(GOCMD) test -BINARY_NAME=clickhouse-diagnostics -BUILD_DIR=dist - -TIMESTAMP := $(shell date +%Y%m%d-%H%M) -COMMIT := $(shell git rev-parse --short HEAD) -MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics -VERSION := v.dev-${TIMESTAMP} -DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" - -# override with env variable to test other versions e.g. 21.11.10.1 -CLICKHOUSE_VERSION ?= latest - -GREEN := $(shell tput -Txterm setaf 2) -YELLOW := $(shell tput -Txterm setaf 3) -WHITE := $(shell tput -Txterm setaf 7) -CYAN := $(shell tput -Txterm setaf 6) -RESET := $(shell tput -Txterm sgr0) - -.PHONY: all test build vendor release lint-go test-coverages dep - -all: help - -release: ## Release is delegated to goreleaser - $(shell goreleaser release --rm-dist) - -## Build: -build: ## Build a binary for local use - # timestamped version - $(GOCMD) build ${DEVLDFLAGS} -o $(BINARY_NAME) ./cmd/clickhouse-diagnostics - -clean: ## Remove build related file - rm ${BINARY_NAME} - rm -f checkstyle-report.xml ./coverage.xml ./profile.cov - -vendor: ## Copy of all packages needed to support builds and tests in the vendor directory - $(GOCMD) mod vendor - -test: ## Run the tests of the project - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race `go list ./... | grep -v ./internal/platform/test` - -test-no-docker: ## Don't run tests depending on dockerd - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race -tags no_docker `go list ./... | grep -v ./internal/platform/test` - -lint-go: ## Use golintci-lint - docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest-alpine golangci-lint run - -test-coverage: ## Run the tests of the project and export the coverage - CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -cover -covermode=count -coverprofile=profile.cov `go list ./... | grep -v ./internal/platform/test` - $(GOCMD) tool cover -func profile.cov - -dep: - $(shell go mod download) - -help: ## Show this help. - @echo '' - @echo 'Usage:' - @echo ' ${YELLOW}make${RESET} ${GREEN}${RESET}' - @echo '' - @echo 'Targets:' - @awk 'BEGIN {FS = ":.*?## "} { \ - if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf " ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \ - else if (/^## .*$$/) {printf " ${CYAN}%s${RESET}\n", substr($$1,4)} \ - }' $(MAKEFILE_LIST) diff --git a/programs/diagnostics/README.md b/programs/diagnostics/README.md deleted file mode 100644 index f800bb0648e..00000000000 --- a/programs/diagnostics/README.md +++ /dev/null @@ -1,167 +0,0 @@ -# Clickhouse Diagnostics Tool - -## Purpose - -This tool provides a means of obtaining a diagnostic bundle from a ClickHouse instance. This bundle can be provided to your nearest ClickHouse support provider in order to assist with the diagnosis of issues. - -## Design Philosophy - -- **No local dependencies** to run. We compile to a platform-independent binary, hence Go. -- **Minimize resource overhead**. Improvements always welcome. -- **Extendable framework**. At its core, the tool provides collectors and outputs. Collectors are independent and are responsible for collecting a specific dataset e.g. system configuration. Outputs produce the diagnostic bundle in a specific format. It should be trivial to add both for contributors. See [Collectors](#collectors) and [Outputs](#outputs) for more details. -- **Convertible output formats**. Outputs produce diagnostic bundles in different formats e.g. archive, simple report etc. Where possible, it should be possible to convert between these formats. For example, an administrator may provide a bundle as an archive to their support provider who in turn wishes to visualise this as a report or even in ClickHouse itself... -- **Something is better than nothing**. Collectors execute independently. We never fail a collection because one fails - preferring to warn the user only. There are good reasons for a collector failure e.g. insufficient permissions or missing data. -- **Execute anywhere** - Ideally, this tool is executed on a ClickHouse host. Some collectors e.g. configuration file collection or system information, rely on this. However, collectors will obtain as much information remotely from the database as possible if executed remotely from the cluster - warning where collection fails. **We do currently require ClickHouse to be running, connecting over the native port**. - -We recommend reading [Permissions, Warnings & Locality](#permissions-warnings--locality). - -## Usage - -### Collection - -The `collect` command allows the collection of a diagnostic bundle. In its simplest form, assuming ClickHouse is running locally on default ports with no password: - -```bash -clickhouse-diagnostics collect -``` - -This will use the default collectors and the simple output. This output produces a timestamped archive bundle in `gz` format in a sub folder named after the host. This folder name can be controlled via the parameter `--id` or configured directly for the simple output parameter `output.simple.folder` (this allows a specific directory to be specified). - -Collectors, Outputs and ClickHouse connection credentials can be specified as shown below: - -```bash -clickhouse-diagnostics collect --password random --username default --collector=system_db,system --output=simple --id my_cluster_name -``` - -This collects the system database and host information from the cluster running locally. The archive bundle will be produced under a folder `my_cluster_name`. - -For further details, use the in built help (the commands below are equivalent): - -```bash -clickhouse-diagnostics collect --help -./clickhouse-diagnostics help collect -``` - -### Help & Finding parameters for collectors & outputs - -Collectors and outputs have their own parameters not listed under the help for the command for the `collect` command. These can be identified using the `help` command. Specifically, - -For more information about a specific collector. - -```bash -Use "clickhouse-diagnostics help --collector [collector]" -``` - -For more information about a specific output. - -```bash -Use "clickhouse-diagnostics help --output [output]" -``` - -### Convert - -Coming soon to a cluster near you... - -## Collectors - -We currently support the following collectors. A `*` indicates this collector is enabled by default: - -- `system_db*` - Collects all tables in the system database, except those which have been excluded and up to a specified row limit. -- `system*` - Collects summary OS and hardware statistics for the host. -- `config*` - Collects the ClickHouse configuration from the local filesystem. A best effort is made using process information if ClickHouse is not installed locally. `include_path` are also considered. -- `db_logs*` - Collects the ClickHouse logs directly from the database. -- `logs*` - Collects the ClickHouse logs directly from the database. -- `summary*` - Collects summary statistics on the database based on a set of known useful queries. This represents the easiest collector to extend - contributions are welcome to this set which can be found [here](https://github.com/ClickHouse/ClickHouse/blob/master/programs/diagnostics/internal/collectors/clickhouse/queries.json). -- `file` - Collects files based on glob patterns. Does not collect directories. To preview files which will be collected try, `clickhouse-diagnostics collect --collectors=file --collector.file.file_pattern= --output report` -- `command` - Collects the output of a user specified command. To preview output, `clickhouse-diagnostics collect --collectors=command --collector.command.command="" --output report` -- `zookeeper_db` - Collects information about zookeeper using the `system.zookeeper` table, recursively iterating the zookeeper tree/table. Note: changing the default parameter values can cause extremely high load to be placed on the database. Use with caution. By default, uses the glob `/clickhouse/{task_queue}/**` to match zookeeper paths and iterates to a max depth of 8. - -## Outputs - -We currently support the following outputs. The `simple` output is currently the default: - -- `simple` - Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. -- `report` - Writes out the diagnostic bundle to the terminal as a simple report. Supports an ascii table format or markdown. -- `clickhouse` - **Under development**. This will allow a bundle to be stored in a cluster allowing visualization in common tooling e.g. Grafana. - -## Simple Output - -Since the `simple` output is the default we provide additional details here. -This output produces a timestamped archive by default in `gz` format under a directory created with either the hostname of the specified collection `--id`. As shown below, a specific folder can also be specified. Compression can also be disabled, leaving just the contents of the folder: - -```bash -./clickhouse-diagnostics help --output simple - -Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive. - -Usage: - --output=simple [flags] - -Flags: - --output.simple.directory string Directory in which to create dump. Defaults to the current directory. (default "./") - --output.simple.format string Format of exported files (default "csv") - --output.simple.skip_archive Don't compress output to an archive -``` - -The archive itself contains a folder for each collector. Each collector can potentially produce many discrete sets of data, known as frames. Each of these typically results in a single file within the collector's folder. For example, each query for the `summary` collector results in a correspondingly named file within the `summary` folder. - -## Permissions, Warnings & Locality - -Some collectors either require specific permissions for complete collection or should be executed on a ClickHouse host. We aim to collate these requirements below: - -- `system_db` - This collect aims to collect all tables in the `system` database. Some tables may fail if certain features are not enabled. Specifically,[allow_introspection_functions](https://clickhouse.com/docs/en/operations/settings/settings/#settings-allow_introspection_functions) is required to collect the `stack_traces` table. [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) must be set for the ClickHouse user specified for collection, to permit access to access management tables e.g. `quota_usage`. -- `db_logs`- The ClickHouse user must have access to the tables `query_log`,`query_thread_log` and `text_log`. -- `logs` - The system user under which the tool is executed must have access to the logs directory. It must therefore also be executed on the target ClickHouse server directly for this collector work. In cases where the logs directory is not a default location e.g. `/var/log/clickhouse-server` we will attempt to establish the location from the ClickHouse configuration. This requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. -- `summary`- This collector executes pre-recorded queries. Some of these read tables concerning access management, thus requiring the ClickHouse user to have the [access_management](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) permission. -- `config` - This collector reads and copies the local configuration files. It thus requires permissions to read the configuration files - which in most cases requires specific permissions to be granted to the run user if you are not comfortable executing the tool under sudo or the `clickhouse` user. - -**If a collector cannot collect specific data because of either execution location or permissions, it will log a warning to the terminal.** - -## Logging - -All logs are output to `stderr`. `stdout` is used exclusively for outputs to print information. - -## Configuration file - -In addition to supporting parameters via the command line, a configuration file can be specified via the `--config`, `-f` flag. - -By default, we look for a configuration file `clickhouse-diagnostics.yml` in the same directory as the binary. If not present, we revert to command line flags. - -**Values set via the command line values always take precedence over those in the configuration file.** - -All parameters can be set via the configuration file and can in most cases be converted to a yaml hierarchy, where periods indicate a nesting. For example, - -`--collector.system_db.row_limit=1` - -becomes - -```yaml -collector: - system_db: - row_limit: 1 -``` - -The following exceptions exist to avoid collisions: - -| Command | Parameter | Configuration File | -|---------|------------|--------------------| -| collect | output | collect.output | -| collect | collectors | collect.collectors | - -## FAQ - -1. Does the collector need root permissions? - - No. However, to read some local files e.g. configurations, the tool should be executed as the `clickhouse` user. - -2. What ClickHouse database permissions does the collector need? - - Read permissions on all system tables are required in most cases - although only specific collectors need this. [Access management permissions]((https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting)) will ensure full collection. - -3. Is any processing done on logs for anonimization purposes? - - Currently no. ClickHouse should not log sensitive information to logs e.g. passwords. - -4. Is sensitive information removed from configuration files e.g. passwords? - - Yes. We remove both passwords and hashed passwords. Please raise an issue if you require further information to be anonimized. We appreciate this is a sensitive topic. diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go deleted file mode 100644 index 0a849a9f520..00000000000 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go deleted file mode 100644 index 503d8e41fb7..00000000000 --- a/programs/diagnostics/cmd/collect.go +++ /dev/null @@ -1,159 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/spf13/viper" -) - -var id string -var output = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "simple", -} - -// access credentials -var host string -var port uint16 -var username string -var password string - -var collectorNames = params.StringSliceOptionsVar{ - Options: collectors.GetCollectorNames(false), - Values: collectors.GetCollectorNames(true), -} - -// holds the collector params passed by the cli -var collectorParams params.ParamMap - -// holds the output params passed by the cli -var outputParams params.ParamMap - -const collectHelpTemplate = `Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Global Flags: -{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Additional help topics: - Use "{{.CommandPath}} [command] --help" for more information about a command. - Use "{{.Parent.Name}} help --collector [collector]" for more information about a specific collector. - Use "{{.Parent.Name}} help --output [output]" for more information about a specific output. -` - -func init() { - collectCmd.Flags().StringVar(&id, "id", getHostName(), "Id of diagnostic bundle") - - // access credentials - collectCmd.Flags().StringVar(&host, "host", "localhost", "ClickHouse host") - collectCmd.Flags().Uint16VarP(&port, "port", "p", 9000, "ClickHouse native port") - collectCmd.Flags().StringVarP(&username, "username", "u", "", "ClickHouse username") - collectCmd.Flags().StringVar(&password, "password", "", "ClickHouse password") - // collectors and outputs - collectCmd.Flags().VarP(&output, "output", "o", fmt.Sprintf("Output Format for the diagnostic Bundle, options: [%s]\n", strings.Join(output.Options, ","))) - collectCmd.Flags().VarP(&collectorNames, "collectors", "c", fmt.Sprintf("Collectors to use, options: [%s]\n", strings.Join(collectorNames.Options, ","))) - - collectorConfigs, err := collectors.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build collector configurations") - } - collectorParams = params.NewParamMap(collectorConfigs) - - outputConfigs, err := outputs.BuildConfigurationOptions() - if err != nil { - log.Fatal().Err(err).Msg("Unable to build output configurations") - } - params.AddParamMapToCmd(collectorParams, collectCmd, "collector", true) - - outputParams = params.NewParamMap(outputConfigs) - params.AddParamMapToCmd(outputParams, collectCmd, "output", true) - - collectCmd.SetFlagErrorFunc(handleFlagErrors) - collectCmd.SetHelpTemplate(collectHelpTemplate) - rootCmd.AddCommand(collectCmd) -} - -var collectCmd = &cobra.Command{ - Use: "collect", - Short: "Collect a diagnostic bundle", - Long: `Collect a ClickHouse diagnostic bundle for a specified ClickHouse instance`, - PreRun: func(cmd *cobra.Command, args []string) { - bindFlagsToConfig(cmd) - }, - Example: fmt.Sprintf(`%s collect --username default --collector=%s --output=simple`, rootCmd.Name(), strings.Join(collectorNames.Options[:2], ",")), - Run: func(cmd *cobra.Command, args []string) { - log.Info().Msgf("executing collect command with %v collectors and %s output", collectorNames.Values, output.Value) - outputConfig := params.ConvertParamsToConfig(outputParams)[output.Value] - runConfig := internal.NewRunConfiguration(id, host, port, username, password, output.Value, outputConfig, collectorNames.Values, params.ConvertParamsToConfig(collectorParams)) - internal.Capture(runConfig) - os.Exit(0) - }, -} - -func getHostName() string { - name, err := os.Hostname() - if err != nil { - name = "clickhouse-diagnostics" - } - return name -} - -// these flags are nested under the cmd name in the config file to prevent collisions -var flagsToNest = []string{"output", "collectors"} - -// this saves us binding each command manually to viper -func bindFlagsToConfig(cmd *cobra.Command) { - cmd.Flags().VisitAll(func(f *pflag.Flag) { - err := viper.BindEnv(f.Name, fmt.Sprintf("%s_%s", envPrefix, - strings.ToUpper(strings.Replace(f.Name, ".", "_", -1)))) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - configFlagName := f.Name - if utils.Contains(flagsToNest, f.Name) { - configFlagName = fmt.Sprintf("%s.%s", cmd.Use, configFlagName) - } - err = viper.BindPFlag(configFlagName, f) - if err != nil { - log.Error().Msgf("Unable to bind %s to config", f.Name) - } - // here we prefer the config value when the param is not set on the cmd line - if !f.Changed && viper.IsSet(configFlagName) { - val := viper.Get(configFlagName) - log.Debug().Msgf("Setting parameter %s from configuration file", f.Name) - err = cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val)) - if err != nil { - log.Error().Msgf("Unable to read \"%s\" value from config", f.Name) - } else { - log.Debug().Msgf("Set parameter \"%s\" from configuration", f.Name) - } - } - }) -} diff --git a/programs/diagnostics/cmd/convert.go b/programs/diagnostics/cmd/convert.go deleted file mode 100644 index 1d619dd05e2..00000000000 --- a/programs/diagnostics/cmd/convert.go +++ /dev/null @@ -1 +0,0 @@ -package cmd diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go deleted file mode 100644 index 750576dda25..00000000000 --- a/programs/diagnostics/cmd/help.go +++ /dev/null @@ -1,124 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -var cHelp = params.StringOptionsVar{ - Options: collectors.GetCollectorNames(false), - Value: "", -} -var oHelp = params.StringOptionsVar{ - Options: outputs.GetOutputNames(), - Value: "", -} - -func init() { - helpCmd.Flags().VarP(&cHelp, "collector", "c", "Specify collector to get description of available flags") - helpCmd.Flags().VarP(&oHelp, "output", "o", "Specify output to get description of available flags") - helpCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Available Commands:{{range .Parent.Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}} - -Alternatively use "{{.CommandPath}} [command] --help" for more information about a command. -`) - helpCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -var helpCmd = &cobra.Command{ - Use: "help [command]", - Short: "Help about any command, collector or output", - Long: `Help provides help for any command, collector or output in the application.`, - Example: fmt.Sprintf(`%[1]v help collect -%[1]v help --collector=config -%[1]v help --output=simple`, rootCmd.Name()), - Run: func(c *cobra.Command, args []string) { - if len(args) != 0 { - //find the command on which help is requested - cmd, _, e := c.Root().Find(args) - if cmd == nil || e != nil { - c.Printf("Unknown help topic %#q\n", args) - cobra.CheckErr(c.Root().Usage()) - } else { - cmd.InitDefaultHelpFlag() - cobra.CheckErr(cmd.Help()) - } - return - } - if cHelp.Value != "" && oHelp.Value != "" { - log.Error().Msg("Specify either --collector or --output not both") - _ = c.Help() - os.Exit(1) - } - if cHelp.Value != "" { - collector, err := collectors.GetCollectorByName(cHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize collector %s", cHelp.Value) - } - configHelp(collector.Configuration(), "collector", cHelp.Value, collector.Description()) - } else if oHelp.Value != "" { - output, err := outputs.GetOutputByName(oHelp.Value) - if err != nil { - log.Fatal().Err(err).Msgf("Unable to initialize output %s", oHelp.Value) - } - configHelp(output.Configuration(), "output", oHelp.Value, output.Description()) - } else { - _ = c.Help() - } - os.Exit(0) - }, -} - -func configHelp(conf config.Configuration, componentType, name, description string) { - - paramMap := params.NewParamMap(map[string]config.Configuration{ - name: conf, - }) - tempHelpCmd := &cobra.Command{ - Use: fmt.Sprintf("--%s=%s", componentType, name), - Short: fmt.Sprintf("Help about the %s collector", name), - Long: description, - SilenceErrors: true, - Run: func(c *cobra.Command, args []string) { - _ = c.Help() - }, - } - params.AddParamMapToCmd(paramMap, tempHelpCmd, componentType, false) - // this is workaround to hide the help flag - tempHelpCmd.Flags().BoolP("help", "h", false, "Dummy help") - tempHelpCmd.Flags().Lookup("help").Hidden = true - tempHelpCmd.SetUsageTemplate(` -{{.Long}} - -Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}} - -Flags:{{if .HasAvailableLocalFlags}} -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{else}} - - No configuration flags available -{{end}} -`) - - _ = tempHelpCmd.Execute() -} diff --git a/programs/diagnostics/cmd/params/params.go b/programs/diagnostics/cmd/params/params.go deleted file mode 100644 index c4464aab5d2..00000000000 --- a/programs/diagnostics/cmd/params/params.go +++ /dev/null @@ -1,281 +0,0 @@ -package params - -import ( - "bytes" - "encoding/csv" - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/spf13/cobra" -) - -type cliParamType uint8 - -const ( - String cliParamType = iota - StringList - StringOptionsList - Integer - Boolean -) - -type CliParam struct { - Description string - Default interface{} - //this should always be an address to a value - as required by cobra - Value interface{} - Type cliParamType -} - -type ParamMap map[string]map[string]CliParam - -func NewParamMap(configs map[string]config.Configuration) ParamMap { - paramMap := make(ParamMap) - for name, configuration := range configs { - for _, param := range configuration.Params { - switch p := param.(type) { - case config.StringParam: - paramMap = paramMap.createStringParam(name, p) - case config.StringListParam: - paramMap = paramMap.createStringListParam(name, p) - case config.StringOptions: - paramMap = paramMap.createStringOptionsParam(name, p) - case config.IntParam: - paramMap = paramMap.createIntegerParam(name, p) - case config.BoolParam: - paramMap = paramMap.createBoolParam(name, p) - } - } - } - return paramMap -} - -func (m ParamMap) createBoolParam(rootKey string, bParam config.BoolParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value bool - param := CliParam{ - Description: bParam.Description(), - Default: bParam.Value, - Value: &value, - Type: Boolean, - } - m[rootKey][bParam.Name()] = param - return m -} - -func (m ParamMap) createStringParam(rootKey string, sParam config.StringParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value string - param := CliParam{ - Description: sParam.Description(), - Default: sParam.Value, - Value: &value, - Type: String, - } - m[rootKey][sParam.Name()] = param - return m -} - -func (m ParamMap) createStringListParam(rootKey string, lParam config.StringListParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value []string - param := CliParam{ - Description: lParam.Description(), - Default: lParam.Values, - Value: &value, - Type: StringList, - } - m[rootKey][lParam.Name()] = param - return m -} - -func (m ParamMap) createStringOptionsParam(rootKey string, oParam config.StringOptions) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - value := StringOptionsVar{ - Options: oParam.Options, - Value: oParam.Value, - } - param := CliParam{ - Description: oParam.Description(), - Default: oParam.Value, - Value: &value, - Type: StringOptionsList, - } - m[rootKey][oParam.Name()] = param - return m -} - -func (m ParamMap) createIntegerParam(rootKey string, iParam config.IntParam) ParamMap { - if _, ok := m[rootKey]; !ok { - m[rootKey] = make(map[string]CliParam) - } - var value int64 - param := CliParam{ - Description: iParam.Description(), - Default: iParam.Value, - Value: &value, - Type: Integer, - } - m[rootKey][iParam.Name()] = param - return m -} - -func (c CliParam) GetConfigParam(name string) config.ConfigParam { - // this is a config being passed to a collector - required can be false - param := config.NewParam(name, c.Description, false) - switch c.Type { - case String: - return config.StringParam{ - Param: param, - // values will be pointers - Value: *(c.Value.(*string)), - } - case StringList: - return config.StringListParam{ - Param: param, - Values: *(c.Value.(*[]string)), - } - case StringOptionsList: - optionsVar := *(c.Value.(*StringOptionsVar)) - return config.StringOptions{ - Param: param, - Options: optionsVar.Options, - Value: optionsVar.Value, - } - case Integer: - return config.IntParam{ - Param: param, - Value: *(c.Value.(*int64)), - } - case Boolean: - return config.BoolParam{ - Param: param, - Value: *(c.Value.(*bool)), - } - } - return param -} - -type StringOptionsVar struct { - Options []string - Value string -} - -func (o StringOptionsVar) String() string { - return o.Value -} - -func (o *StringOptionsVar) Set(p string) error { - isIncluded := func(opts []string, val string) bool { - for _, opt := range opts { - if val == opt { - return true - } - } - return false - } - if !isIncluded(o.Options, p) { - return fmt.Errorf("%s is not included in options: %v", p, o.Options) - } - o.Value = p - return nil -} - -func (o *StringOptionsVar) Type() string { - return "string" -} - -type StringSliceOptionsVar struct { - Options []string - Values []string -} - -func (o StringSliceOptionsVar) String() string { - str, _ := writeAsCSV(o.Values) - return "[" + str + "]" -} - -func (o *StringSliceOptionsVar) Set(val string) error { - values, err := readAsCSV(val) - if err != nil { - return err - } - vValues := utils.Distinct(values, o.Options) - if len(vValues) > 0 { - return fmt.Errorf("%v are not included in options: %v", vValues, o.Options) - } - o.Values = values - return nil -} - -func (o *StringSliceOptionsVar) Type() string { - return "stringSlice" -} - -func writeAsCSV(vals []string) (string, error) { - b := &bytes.Buffer{} - w := csv.NewWriter(b) - err := w.Write(vals) - if err != nil { - return "", err - } - w.Flush() - return strings.TrimSuffix(b.String(), "\n"), nil -} - -func readAsCSV(val string) ([]string, error) { - if val == "" { - return []string{}, nil - } - stringReader := strings.NewReader(val) - csvReader := csv.NewReader(stringReader) - return csvReader.Read() -} - -func AddParamMapToCmd(paramMap ParamMap, cmd *cobra.Command, prefix string, hide bool) { - for rootKey, childMap := range paramMap { - for childKey, value := range childMap { - paramName := fmt.Sprintf("%s.%s.%s", prefix, rootKey, childKey) - switch value.Type { - case String: - cmd.Flags().StringVar(value.Value.(*string), paramName, value.Default.(string), value.Description) - case StringList: - cmd.Flags().StringSliceVar(value.Value.(*[]string), paramName, value.Default.([]string), value.Description) - case StringOptionsList: - cmd.Flags().Var(value.Value.(*StringOptionsVar), paramName, value.Description) - case Integer: - cmd.Flags().Int64Var(value.Value.(*int64), paramName, value.Default.(int64), value.Description) - case Boolean: - cmd.Flags().BoolVar(value.Value.(*bool), paramName, value.Default.(bool), value.Description) - } - // this ensures flags from collectors and outputs are not shown as they will pollute the output - if hide { - _ = cmd.Flags().MarkHidden(paramName) - } - } - } -} - -func ConvertParamsToConfig(paramMap ParamMap) map[string]config.Configuration { - configuration := make(map[string]config.Configuration) - for rootKey, childMap := range paramMap { - if _, ok := configuration[rootKey]; !ok { - configuration[rootKey] = config.Configuration{} - } - for childKey, value := range childMap { - configParam := value.GetConfigParam(childKey) - configuration[rootKey] = config.Configuration{Params: append(configuration[rootKey].Params, configParam)} - } - } - return configuration -} diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go deleted file mode 100644 index 7671506ba59..00000000000 --- a/programs/diagnostics/cmd/params/params_test.go +++ /dev/null @@ -1,247 +0,0 @@ -package params_test - -import ( - "os" - "sort" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" -) - -var conf = map[string]config.Configuration{ - "config": { - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "A directory", false), - AllowEmpty: true, - }, - }, - }, - "system": { - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Include tables", false), - }, - config.StringListParam{ - Values: []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, - Param: config.NewParam("exclude_tables", "Excluded tables", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Max rows", false), - }, - }, - }, - "reader": { - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of imported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect archives", false), - }, - }, - }, -} - -func TestNewParamMap(t *testing.T) { - // test each of the types via NewParamMap - one with each type. the keys here can represent anything e.g. a collector name - t.Run("test param map correctly converts types", func(t *testing.T) { - paramMap := params.NewParamMap(conf) - require.Len(t, paramMap, 3) - // check config - require.Contains(t, paramMap, "config") - require.Len(t, paramMap["config"], 1) - require.Contains(t, paramMap["config"], "directory") - require.IsType(t, params.CliParam{}, paramMap["config"]["directory"]) - require.Equal(t, "A directory", paramMap["config"]["directory"].Description) - require.Equal(t, "", *(paramMap["config"]["directory"].Value.(*string))) - require.Equal(t, "", paramMap["config"]["directory"].Default) - require.Equal(t, params.String, paramMap["config"]["directory"].Type) - // check system - require.Contains(t, paramMap, "system") - require.Len(t, paramMap["system"], 3) - require.IsType(t, params.CliParam{}, paramMap["system"]["include_tables"]) - - require.Equal(t, "Include tables", paramMap["system"]["include_tables"].Description) - var value []string - require.Equal(t, &value, paramMap["system"]["include_tables"].Value) - require.Equal(t, value, paramMap["system"]["include_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["include_tables"].Type) - - require.Equal(t, "Excluded tables", paramMap["system"]["exclude_tables"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["exclude_tables"]) - require.Equal(t, &value, paramMap["system"]["exclude_tables"].Value) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, paramMap["system"]["exclude_tables"].Default) - require.Equal(t, params.StringList, paramMap["system"]["exclude_tables"].Type) - - require.Equal(t, "Max rows", paramMap["system"]["row_limit"].Description) - require.IsType(t, params.CliParam{}, paramMap["system"]["row_limit"]) - var iValue int64 - require.Equal(t, &iValue, paramMap["system"]["row_limit"].Value) - require.Equal(t, int64(100000), paramMap["system"]["row_limit"].Default) - require.Equal(t, params.Integer, paramMap["system"]["row_limit"].Type) - - // check reader - require.Contains(t, paramMap, "reader") - require.Len(t, paramMap["reader"], 2) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - require.Equal(t, "Format of imported files", paramMap["reader"]["format"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["format"]) - oValue := params.StringOptionsVar{ - Options: []string{"csv"}, - Value: "csv", - } - require.Equal(t, &oValue, paramMap["reader"]["format"].Value) - require.Equal(t, "csv", paramMap["reader"]["format"].Default) - require.Equal(t, params.StringOptionsList, paramMap["reader"]["format"].Type) - - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - require.Equal(t, "Collect archives", paramMap["reader"]["collect_archives"].Description) - require.IsType(t, params.CliParam{}, paramMap["reader"]["collect_archives"]) - var bVar bool - require.Equal(t, &bVar, paramMap["reader"]["collect_archives"].Value) - require.Equal(t, true, paramMap["reader"]["collect_archives"].Default) - require.Equal(t, params.Boolean, paramMap["reader"]["collect_archives"].Type) - - }) - -} - -// test GetConfigParam -func TestConvertParamsToConfig(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can convert a param map back to a config", func(t *testing.T) { - cParam := params.ConvertParamsToConfig(paramMap) - // these will not be equal as we have some information loss e.g. allowEmpty - //require.Equal(t, conf, cParam) - // deep equality - for name := range conf { - require.Equal(t, len(conf[name].Params), len(cParam[name].Params)) - // sort both consistently - sort.Slice(conf[name].Params, func(i, j int) bool { - return conf[name].Params[i].Name() < conf[name].Params[j].Name() - }) - sort.Slice(cParam[name].Params, func(i, j int) bool { - return cParam[name].Params[i].Name() < cParam[name].Params[j].Name() - }) - for i, param := range conf[name].Params { - require.Equal(t, param.Required(), cParam[name].Params[i].Required()) - require.Equal(t, param.Name(), cParam[name].Params[i].Name()) - require.Equal(t, param.Description(), cParam[name].Params[i].Description()) - } - } - }) -} - -// create via NewParamMap and add to command AddParamMapToCmd - check contents -func TestAddParamMapToCmd(t *testing.T) { - paramMap := params.NewParamMap(conf) - t.Run("test we can add hidden params to a command", func(t *testing.T) { - testComand := &cobra.Command{ - Use: "test", - Short: "Run a test", - Long: `Longer description`, - Run: func(cmd *cobra.Command, args []string) { - os.Exit(0) - }, - } - params.AddParamMapToCmd(paramMap, testComand, "collector", true) - // check we get an error on one which doesn't exist - _, err := testComand.Flags().GetString("collector.config.random") - require.NotNil(t, err) - // check getting incorrect type - _, err = testComand.Flags().GetString("collector.system.include_tables") - require.NotNil(t, err) - - // check existence of all flags - directory, err := testComand.Flags().GetString("collector.config.directory") - require.Nil(t, err) - require.Equal(t, "", directory) - - includeTables, err := testComand.Flags().GetStringSlice("collector.system.include_tables") - require.Nil(t, err) - require.Equal(t, []string{}, includeTables) - - excludeTables, err := testComand.Flags().GetStringSlice("collector.system.exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper"}, excludeTables) - - rowLimit, err := testComand.Flags().GetInt64("collector.system.row_limit") - require.Nil(t, err) - require.Equal(t, int64(100000), rowLimit) - - format, err := testComand.Flags().GetString("collector.reader.format") - require.Nil(t, err) - require.Equal(t, "csv", format) - - collectArchives, err := testComand.Flags().GetBool("collector.reader.collect_archives") - require.Nil(t, err) - require.Equal(t, true, collectArchives) - }) -} - -// test StringOptionsVar -func TestStringOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("tsv") - require.Nil(t, err) - require.Equal(t, "tsv", format.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - format := params.StringOptionsVar{ - Options: []string{"csv", "tsv", "native"}, - Value: "csv", - } - require.Equal(t, "csv", format.String()) - err := format.Set("random") - require.NotNil(t, err) - require.Equal(t, "random is not included in options: [csv tsv native]", err.Error()) - }) -} - -// test StringSliceOptionsVar -func TestStringSliceOptionsVar(t *testing.T) { - - t.Run("test we can set", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,native") - require.Nil(t, err) - require.Equal(t, "[tsv,native]", formats.String()) - }) - - t.Run("test set invalid", func(t *testing.T) { - formats := params.StringSliceOptionsVar{ - Options: []string{"csv", "tsv", "native", "qsv"}, - Values: []string{"csv", "tsv"}, - } - require.Equal(t, "[csv,tsv]", formats.String()) - err := formats.Set("tsv,random") - require.NotNil(t, err) - require.Equal(t, "[random] are not included in options: [csv tsv native qsv]", err.Error()) - err = formats.Set("msv,random") - require.NotNil(t, err) - require.Equal(t, "[msv random] are not included in options: [csv tsv native qsv]", err.Error()) - }) - -} diff --git a/programs/diagnostics/cmd/root.go b/programs/diagnostics/cmd/root.go deleted file mode 100644 index 4cf329d5438..00000000000 --- a/programs/diagnostics/cmd/root.go +++ /dev/null @@ -1,174 +0,0 @@ -package cmd - -import ( - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "strings" - "time" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func enableDebug() { - if debug { - zerolog.SetGlobalLevel(zerolog.DebugLevel) - go func() { - err := http.ListenAndServe("localhost:8080", nil) - if err != nil { - log.Error().Err(err).Msg("unable to start debugger") - } else { - log.Debug().Msg("debugger has been started on port 8080") - } - }() - } -} - -var rootCmd = &cobra.Command{ - Use: "clickhouse-diagnostics", - Short: "Capture and convert ClickHouse diagnostic bundles.", - Long: `Captures ClickHouse diagnostic bundles to a number of supported formats, including file and ClickHouse itself. Converts bundles between formats.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - enableDebug() - err := initializeConfig() - if err != nil { - log.Error().Err(err) - os.Exit(1) - } - return nil - }, - Example: `clickhouse-diagnostics collect`, -} - -const ( - colorRed = iota + 31 - colorGreen - colorYellow - colorMagenta = 35 - - colorBold = 1 -) - -const TimeFormat = time.RFC3339 - -var debug bool -var configFiles []string - -const ( - // The environment variable prefix of all environment variables bound to our command line flags. - // For example, --output is bound to CLICKHOUSE_DIAGNOSTIC_OUTPUT. - envPrefix = "CLICKHOUSE_DIAGNOSTIC" -) - -func init() { - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug mode") - rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "f", []string{"clickhouse-diagnostics.yml", "/etc/clickhouse-diagnostics.yml"}, "Configuration file path") - // set a usage template to ensure flags on root are listed as global - rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}} - {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} - {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} - -Aliases: - {{.NameAndAliases}}{{end}}{{if .HasExample}} - -Examples: -{{.Example}}{{end}}{{if .HasAvailableSubCommands}} - -Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} - {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} - -Global Flags: -{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} - -Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} - {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} - -Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} -`) - rootCmd.SetFlagErrorFunc(handleFlagErrors) - -} - -func Execute() { - // logs go to stderr - stdout is exclusive for outputs e.g. tables - output := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: TimeFormat} - // override the colors - output.FormatLevel = func(i interface{}) string { - var l string - if ll, ok := i.(string); ok { - switch ll { - case zerolog.LevelTraceValue: - l = colorize("TRC", colorMagenta) - case zerolog.LevelDebugValue: - l = colorize("DBG", colorMagenta) - case zerolog.LevelInfoValue: - l = colorize("INF", colorGreen) - case zerolog.LevelWarnValue: - l = colorize(colorize("WRN", colorYellow), colorBold) - case zerolog.LevelErrorValue: - l = colorize(colorize("ERR", colorRed), colorBold) - case zerolog.LevelFatalValue: - l = colorize(colorize("FTL", colorRed), colorBold) - case zerolog.LevelPanicValue: - l = colorize(colorize("PNC", colorRed), colorBold) - default: - l = colorize("???", colorBold) - } - } else { - if i == nil { - l = colorize("???", colorBold) - } else { - l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] - } - } - return l - } - output.FormatTimestamp = func(i interface{}) string { - tt := i.(string) - return colorize(tt, colorGreen) - } - log.Logger = log.Output(output) - zerolog.SetGlobalLevel(zerolog.InfoLevel) - rootCmd.SetHelpCommand(helpCmd) - if err := rootCmd.Execute(); err != nil { - log.Fatal().Err(err) - } -} - -// colorize returns the string s wrapped in ANSI code c -func colorize(s interface{}, c int) string { - return fmt.Sprintf("\x1b[%dm%v\x1b[0m", c, s) -} - -func handleFlagErrors(cmd *cobra.Command, err error) error { - fmt.Println(colorize(colorize(fmt.Sprintf("Error: %s\n", err), colorRed), colorBold)) - _ = cmd.Help() - os.Exit(1) - return nil -} - -func initializeConfig() error { - // we use the first config file we find - var configFile string - for _, confFile := range configFiles { - if ok, _ := utils.FileExists(confFile); ok { - configFile = confFile - break - } - } - if configFile == "" { - log.Warn().Msgf("config file in %s not found - config file will be ignored", configFiles) - return nil - } - viper.SetConfigFile(configFile) - if err := viper.ReadInConfig(); err != nil { - return errors.Wrapf(err, "Unable to read configuration file at %s", configFile) - } - return nil -} diff --git a/programs/diagnostics/cmd/version.go b/programs/diagnostics/cmd/version.go deleted file mode 100644 index b1c0b44171b..00000000000 --- a/programs/diagnostics/cmd/version.go +++ /dev/null @@ -1,24 +0,0 @@ -package cmd - -import ( - "fmt" - "github.com/spf13/cobra" -) - -var ( - Version = "" // set at compile time with -ldflags "-X versserv/cmd.Version=x.y.yz" - Commit = "" -) - -func init() { - rootCmd.AddCommand(versionCmd) -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "Print the version number of clickhouse-diagnostics", - Long: `All software has versions. This is clickhouse-diagnostics`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Clickhouse Diagnostics %s (%s)\n", Version, Commit) - }, -} diff --git a/programs/diagnostics/go.mod b/programs/diagnostics/go.mod deleted file mode 100644 index 34c6b0037ae..00000000000 --- a/programs/diagnostics/go.mod +++ /dev/null @@ -1,89 +0,0 @@ -module github.com/ClickHouse/ClickHouse/programs/diagnostics - -go 1.19 - -require ( - github.com/ClickHouse/clickhouse-go/v2 v2.0.12 - github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/Masterminds/semver v1.5.0 - github.com/bmatcuk/doublestar/v4 v4.0.2 - github.com/docker/go-connections v0.4.0 - github.com/elastic/gosigar v0.14.2 - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/jaypipes/ghw v0.8.0 - github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f - github.com/mholt/archiver/v4 v4.0.0-alpha.4 - github.com/olekukonko/tablewriter v0.0.5 - github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.26.1 - github.com/spf13/cobra v1.3.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.8.1 - github.com/testcontainers/testcontainers-go v0.18.0 - github.com/yargevad/filepathx v1.0.0 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect - github.com/containerd/containerd v1.6.17 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/distribution v2.8.2+incompatible // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v23.0.0+incompatible // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/dsnet/compress v0.0.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-ole/go-ole v1.2.4 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jaypipes/pcidb v0.6.0 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/moby/patternmatcher v0.5.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect - github.com/opencontainers/runc v1.1.3 // indirect - github.com/paulmach/orb v0.4.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pierrec/lz4/v4 v4.1.14 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/spf13/afero v1.8.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect - go.opentelemetry.io/otel v1.4.1 // indirect - go.opentelemetry.io/otel/trace v1.4.1 // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect - google.golang.org/grpc v1.47.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect -) diff --git a/programs/diagnostics/go.sum b/programs/diagnostics/go.sum deleted file mode 100644 index a95dfb4fd2b..00000000000 --- a/programs/diagnostics/go.sum +++ /dev/null @@ -1,992 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12 h1:Nbl/NZwoM6LGJm7smNBgvtdr/rxjlIssSW3eG/Nmb9E= -github.com/ClickHouse/clickhouse-go/v2 v2.0.12/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= -github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= -github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.17 h1:XDnJIeJW0cLf6v7/+N+6L9kGrChHeXekZp2VHu6OpiY= -github.com/containerd/containerd v1.6.17/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/distribution v2.8.2+incompatible h1:k9+4DKdOG+quPFZXT/mUsiQrGu9vYCp+dXpuPkuqhk8= -github.com/distribution/distribution v2.8.2+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v23.0.0+incompatible h1:L6c28tNyqZ4/ub9AZC9d5QUuunoHHfEH4/Ue+h/E5nE= -github.com/docker/docker v23.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jaypipes/ghw v0.8.0 h1:02q1pTm9CD83vuhBsEZZhOCS128pq87uyaQeJZkp3sQ= -github.com/jaypipes/ghw v0.8.0/go.mod h1:+gR9bjm3W/HnFi90liF+Fj9GpCe/Dsibl9Im8KmC7c4= -github.com/jaypipes/pcidb v0.6.0 h1:VIM7GKVaW4qba30cvB67xSCgJPTzkG8Kzw/cbs5PHWU= -github.com/jaypipes/pcidb v0.6.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f h1:B0OD7nYl2FPQEVrw8g2uyc1lGEzNbvrKh7fspGZcbvY= -github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f/go.mod h1:aEt7p9Rvh67BYApmZwNDPpgircTO2kgdmDUoF/1QmwA= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/archiver/v4 v4.0.0-alpha.4 h1:QJ4UuWgavPynEX3LXxClHDRGzYcgcvTtAMp8az7spuw= -github.com/mholt/archiver/v4 v4.0.0-alpha.4/go.mod h1:J7SYS/UTAtnO3I49RQEf+2FYZVwo7XBOh9Im43VrjNs= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= -github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f h1:J/7hjLaHLD7epG0m6TBMGmp4NQ+ibBYLfeyJWdAIFLA= -github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f/go.mod h1:15ce4BGCFxt7I5NQKT+HV0yEDxmf6fSysfEDiVo3zFM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= -github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= -github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= -github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= -github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/testcontainers/testcontainers-go v0.18.0 h1:8RXrcIQv5xX/uBOSmZd297gzvA7F0yuRA37/918o7Yg= -github.com/testcontainers/testcontainers-go v0.18.0/go.mod h1:rLC7hR2SWRjJZZNrUYiTKvUXCziNxzZiYtz9icTWYNQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= -github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g= -go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= -go.opentelemetry.io/otel/trace v1.4.1 h1:O+16qcdTrT7zxv2J6GejTPFinSwA++cYerC5iSiF8EQ= -go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go deleted file mode 100644 index 92368bce6f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ /dev/null @@ -1,113 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type ConfigCollector struct { - resourceManager *platform.ResourceManager -} - -func NewConfigCollector(m *platform.ResourceManager) *ConfigCollector { - return &ConfigCollector{ - resourceManager: m, - } -} - -const DefaultConfigLocation = "/etc/clickhouse-server/" -const ProcessedConfigurationLocation = "/var/lib/clickhouse/preprocessed_configs" - -func (c ConfigCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the config - frame, errs := data.NewConfigFileFrame(directory) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - configCandidates, err := FindConfigurationFiles() - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to find configuration files") - } - frames := make(map[string]data.Frame) - var frameErrors []error - for frameName, confDir := range configCandidates { - frame, errs := data.NewConfigFileFrame(confDir) - frameErrors = append(frameErrors, errs...) - frames[frameName] = frame - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func FindConfigurationFiles() (map[string]string, error) { - configCandidates := map[string]string{ - "default": DefaultConfigLocation, - "preprocessed": ProcessedConfigurationLocation, - } - // we don't know specifically where the config is but try to find via processes - processConfigs, err := utils.FindConfigsFromClickHouseProcesses() - if err != nil { - return nil, err - } - for i, path := range processConfigs { - confDir := filepath.Dir(path) - if len(processConfigs) == 1 { - configCandidates["process"] = confDir - break - } - configCandidates[fmt.Sprintf("process_%d", i)] = confDir - } - return configCandidates, nil -} - -func (c ConfigCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the configuration files for ClickHouse Server e.g. /etc/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - } -} - -func (c ConfigCollector) Description() string { - return "Collects the ClickHouse configuration from the local filesystem." -} - -func (c ConfigCollector) IsDefault() bool { - return true -} - -// here we register the collector for use -func init() { - collectors.Register("config", func() (collectors.Collector, error) { - return &ConfigCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/config_test.go b/programs/diagnostics/internal/collectors/clickhouse/config_test.go deleted file mode 100644 index 355cbb65620..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/config_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package clickhouse_test - -import ( - "encoding/xml" - "fmt" - "io" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestConfigConfiguration(t *testing.T) { - t.Run("correct configuration is returned for config collector", func(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - conf := configCollector.Configuration() - require.Len(t, conf.Params, 1) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Equal(t, "", directory.Value) - }) -} - -func TestConfigCollect(t *testing.T) { - configCollector := clickhouse.NewConfigCollector(&platform.ResourceManager{}) - - t.Run("test default file collector configuration", func(t *testing.T) { - diagSet, err := configCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - // we won't be able to collect the default configs preprocessed and default - even if clickhouse is installed - // these directories should not be readable under any permissions these tests are unrealistically executed! - // note: we may also pick up configs from a local clickhouse process - we thus allow a len >=2 but don't check this - // as its non-deterministic - require.GreaterOrEqual(t, len(diagSet.Frames), 2) - // check default key - require.Contains(t, diagSet.Frames, "default") - require.Equal(t, diagSet.Frames["default"].Name(), "/etc/clickhouse-server/") - require.Equal(t, diagSet.Frames["default"].Columns(), []string{"config"}) - // collection will have failed - checkFrame(t, diagSet.Frames["default"], nil) - // check preprocessed key - require.Contains(t, diagSet.Frames, "preprocessed") - require.Equal(t, diagSet.Frames["preprocessed"].Name(), "/var/lib/clickhouse/preprocessed_configs") - require.Equal(t, diagSet.Frames["preprocessed"].Columns(), []string{"config"}) - // min of 2 - might be more if a local installation of clickhouse is running - require.GreaterOrEqual(t, len(diagSet.Errors.Errors), 2) - }) - - t.Run("test configuration when specified", func(t *testing.T) { - // create some test files - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - length is 6 for the included file - rows := make([][]interface{}, 6) - for i := 0; i < 5; i++ { - if i == 4 { - // set the includes for the last doc - xmlConfig.IncludeFrom = includesPath - } - // we want to check hierarchies are walked so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - row := make([]interface{}, 1) - row[0] = data.XmlConfigFile{Path: filepath} - rows[i] = row - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlConfig := xmlConfig - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - diagSet, err := configCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: confDir, - Param: config.NewParam("directory", "File locations", false), - }, - }, - }) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "user_specified") - require.Equal(t, diagSet.Frames["user_specified"].Name(), confDir) - require.Equal(t, diagSet.Frames["user_specified"].Columns(), []string{"config"}) - iConf := make([]interface{}, 1) - iConf[0] = data.XmlConfigFile{Path: includesPath, Included: true} - rows[5] = iConf - checkFrame(t, diagSet.Frames["user_specified"], rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go deleted file mode 100644 index 3253f504c1b..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go +++ /dev/null @@ -1,108 +0,0 @@ -package clickhouse - -import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" -) - -type DBLogTable struct { - orderBy data.OrderBy - excludeColumns []string -} - -var DbLogTables = map[string]DBLogTable{ - "query_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "query_thread_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, - "text_log": { - orderBy: data.OrderBy{ - Column: "event_time_microseconds", - Order: data.Asc, - }, - excludeColumns: []string{}, - }, -} - -// This collector collects db logs - -type DBLogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewDBLogsCollector(m *platform.ResourceManager) *DBLogsCollector { - return &DBLogsCollector{ - resourceManager: m, - } -} - -func (dc *DBLogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(dc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - frames := make(map[string]data.Frame) - var frameErrors []error - for logTable, tableConfig := range DbLogTables { - frame, err := dc.resourceManager.DbClient.ReadTable("system", logTable, tableConfig.excludeColumns, tableConfig.orderBy, rowLimit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", logTable)) - } else { - frames[logTable] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (dc *DBLogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } -} - -func (dc *DBLogsCollector) IsDefault() bool { - return true -} - -func (dc DBLogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("db_logs", func() (collectors.Collector, error) { - return &DBLogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go deleted file mode 100644 index 3fc585f3352..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestDbLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := dbLogsCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - rowLimit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, rowLimit.Name(), "row_limit") - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestDbLogsCollect(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - dbLogsCollector := clickhouse.NewDBLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - queryLogColumns := []string{"type", "event_date", "event_time", "event_time_microseconds", - "query_start_time", "query_start_time_microseconds", "query_duration_ms", "read_rows", "read_bytes", "written_rows", "written_bytes", - "result_rows", "result_bytes", "memory_usage", "current_database", "query", "formatted_query", "normalized_query_hash", - "query_kind", "databases", "tables", "columns", "projections", "views", "exception_code", "exception", "stack_trace", - "is_initial_query", "user", "query_id", "address", "port", "initial_user", "initial_query_id", "initial_address", "initial_port", - "initial_query_start_time", "initial_query_start_time_microseconds", "interface", "os_user", "client_hostname", "client_name", - "client_revision", "client_version_major", "client_version_minor", "client_version_patch", "http_method", "http_user_agent", - "http_referer", "forwarded_for", "quota_key", "revision", "log_comment", "thread_ids", "ProfileEvents", "Settings", - "used_aggregate_functions", "used_aggregate_function_combinators", "used_database_engines", "used_data_type_families", - "used_dictionaries", "used_formats", "used_functions", "used_storages", "used_table_functions"} - queryLogFrame := test.NewFakeDataFrame("queryLog", queryLogColumns, - [][]interface{}{ - {"QueryStart", "2021-12-13", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "2021-12-13 12:53:20", "2021-12-13 12:53:20.590579", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 12:53:30", "2021-12-13 12:53:30.607292", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "15", "4512", "255694", "0", "0", "4358", "173248", "4415230", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "default", "3b5feb6d-3086-4718-adb2-17464988ff12", "::ffff:127.0.0.1", "50920", "2021-12-13 12:53:30", "2021-12-13 12:53:30.590579", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95315,95587,95316,95312,95589,95318,95586,95588,95585]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':463,'NetworkSendBytes':88452,'SelectedRows':4512,'SelectedBytes':255694,'RegexpCreated':6,'ContextLock':411,'RWLockAcquiredReadLocks':190,'RealTimeMicroseconds':49221,'UserTimeMicroseconds':19811,'SystemTimeMicroseconds':2817,'SoftPageFaults':1128,'OSCPUWaitMicroseconds':127,'OSCPUVirtualTimeMicroseconds':22624,'OSWriteBytes':12288,'OSWriteChars':13312}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - {"QueryStart", "2021-12-13", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "0", "0", "0", "0", "0", "0", "0", "0", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[]", "{}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]", "[]"}, - {"QueryFinish", "2021-12-13", "2021-12-13 13:02:56", "2021-12-13 13:02:56.437115", "2021-12-13 13:02:56", "2021-12-13 13:02:56.419528", "16", "4629", "258376", "0", "0", "4377", "174272", "4404694", "default", "SELECT DISTINCT arrayJoin(extractAll(name, '[\\w_]{2,}')) AS res FROM (SELECT name FROM system.functions UNION ALL SELECT name FROM system.table_engines UNION ALL SELECT name FROM system.formats UNION ALL SELECT name FROM system.table_functions UNION ALL SELECT name FROM system.data_type_families UNION ALL SELECT name FROM system.merge_tree_settings UNION ALL SELECT name FROM system.settings UNION ALL SELECT cluster FROM system.clusters UNION ALL SELECT macro FROM system.macros UNION ALL SELECT policy_name FROM system.storage_policies UNION ALL SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate UNION ALL SELECT name FROM system.databases LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.tables LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.dictionaries LIMIT 10000 UNION ALL SELECT DISTINCT name FROM system.columns LIMIT 10000) WHERE notEmpty(res)", "", "6666026786019643712", "Select", "['system']", "['system.aggregate_function_combinators','system.clusters','system.columns','system.data_type_families','system.databases','system.dictionaries','system.formats','system.functions','system.macros','system.merge_tree_settings','system.settings','system.storage_policies','system.table_engines','system.table_functions','system.tables']", "['system.aggregate_function_combinators.name','system.clusters.cluster','system.columns.name','system.data_type_families.name','system.databases.name','system.dictionaries.name','system.formats.name','system.functions.is_aggregate','system.functions.name','system.macros.macro','system.merge_tree_settings.name','system.settings.name','system.storage_policies.policy_name','system.table_engines.name','system.table_functions.name','system.tables.name']", "[]", "[]", "0", "", "", "1", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "default", "351b58e4-6128-47d4-a7b8-03d78c1f84c6", "::ffff:127.0.0.1", "50968", "2021-12-13 13:02:53", "2021-12-13 13:02:53.419528", "1", "", "", "ClickHouse client", "54450", "21", "11", "0", "0", "", "", "", "", "54456", "", "[95298,95318,95315,95316,95312,95588,95589,95586,95585,95587]", "{'Query':1,'SelectQuery':1,'ArenaAllocChunks':41,'ArenaAllocBytes':401408,'FunctionExecute':62,'NetworkSendElapsedMicroseconds':740,'NetworkSendBytes':88794,'SelectedRows':4629,'SelectedBytes':258376,'ContextLock':411,'RWLockAcquiredReadLocks':194,'RealTimeMicroseconds':52469,'UserTimeMicroseconds':17179,'SystemTimeMicroseconds':4218,'SoftPageFaults':569,'OSCPUWaitMicroseconds':303,'OSCPUVirtualTimeMicroseconds':25087,'OSWriteBytes':12288,'OSWriteChars':12288}", "{'load_balancing':'random','max_memory_usage':'10000000000'}", "[]", "[]", "[]", "[]", "[]", "[]", "['concat','notEmpty','extractAll']", "[]", "[]"}, - }) - - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &queryLogFrame - - textLogColumns := []string{"event_date", "event_time", "event_time_microseconds", "microseconds", "thread_name", "thread_id", "level", "query_id", "logger_name", "message", "revision", "source_file", "source_line"} - textLogFrame := test.NewFakeDataFrame("textLog", textLogColumns, - [][]interface{}{ - {"2022-02-03", "2022-02-03 16:17:47", "2022-02-03 16:37:17.056950", "56950", "clickhouse-serv", "68947", "Information", "", "DNSCacheUpdater", "Update period 15 seconds", "54458", "../src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()", "46"}, - {"2022-02-03", "2022-02-03 16:27:47", "2022-02-03 16:37:27.057022", "57022", "clickhouse-serv", "68947", "Information", "", "Application", "Available RAM: 62.24 GiB; physical cores: 8; logical cores: 16.", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1380"}, - {"2022-02-03", "2022-02-03 16:37:47", "2022-02-03 16:37:37.057484", "57484", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for http://[::1]:8123", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - {"2022-02-03", "2022-02-03 16:47:47", "2022-02-03 16:37:47.057527", "57527", "clickhouse-serv", "68947", "Information", "", "Application", "Listening for native protocol (tcp): [::1]:9000", "54458", "../programs/server/Server.cpp; virtual int DB::Server::main(const std::vector &)", "1444"}, - }) - - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 100000"] = &textLogFrame - - // skip query_thread_log frame - often it doesn't exist anyway unless enabled - t.Run("test default db logs collection", func(t *testing.T) { - bundle, errs := dbLogsCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - require.Len(t, bundle.Errors.Errors, 1) - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows) - client.Reset() - }) - - t.Run("test db logs collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of log rows to collect. Negative values mean unlimited", false), - }, - }, - } - bundle, err := dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 0) - require.Len(t, bundle.Errors.Errors, 3) - // populate client - client.QueryResponses["SELECT * FROM system.query_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &queryLogFrame - client.QueryResponses["SELECT * FROM system.text_log ORDER BY event_time_microseconds ASC LIMIT 1"] = &textLogFrame - bundle, err = dbLogsCollector.Collect(conf) - require.Empty(t, err) - require.Len(t, bundle.Frames, 2) - require.Len(t, bundle.Errors.Errors, 1) - require.Contains(t, bundle.Frames, "text_log") - require.Contains(t, bundle.Frames, "query_log") - // check query_log frame - require.Contains(t, bundle.Frames, "query_log") - require.Equal(t, queryLogColumns, bundle.Frames["query_log"].Columns()) - checkFrame(t, bundle.Frames["query_log"], queryLogFrame.Rows[:1]) - //check text_log frame - require.Contains(t, bundle.Frames, "text_log") - require.Equal(t, textLogColumns, bundle.Frames["text_log"].Columns()) - checkFrame(t, bundle.Frames["text_log"], textLogFrame.Rows[:1]) - client.Reset() - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs.go b/programs/diagnostics/internal/collectors/clickhouse/logs.go deleted file mode 100644 index 8436a392c47..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs.go +++ /dev/null @@ -1,140 +0,0 @@ -package clickhouse - -import ( - "fmt" - "path/filepath" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -// This collector collects logs - -type LogsCollector struct { - resourceManager *platform.ResourceManager -} - -func NewLogsCollector(m *platform.ResourceManager) *LogsCollector { - return &LogsCollector{ - resourceManager: m, - } -} - -var DefaultLogsLocation = filepath.Clean("/var/log/clickhouse-server/") - -func (lc *LogsCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(lc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return &data.DiagnosticBundle{}, err - } - collectArchives, err := config.ReadBoolValue(conf, "collect_archives") - if err != nil { - return &data.DiagnosticBundle{}, err - } - logPatterns := []string{"*.log"} - if collectArchives { - logPatterns = append(logPatterns, "*.gz") - } - - if directory != "" { - // user has specified a directory - we therefore skip all other efforts to locate the logs - frame, errs := data.NewFileDirectoryFrame(directory, logPatterns) - return &data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "user_specified": frame, - }, - Errors: data.FrameErrors{Errors: errs}, - }, nil - } - // add the default - frames := make(map[string]data.Frame) - dirFrame, frameErrors := data.NewFileDirectoryFrame(DefaultLogsLocation, logPatterns) - frames["default"] = dirFrame - logFolders, errs := FindLogFileCandidates() - frameErrors = append(frameErrors, errs...) - i := 0 - for folder, paths := range logFolders { - // we will collect the default location anyway above so skip these - if folder != DefaultLogsLocation { - if collectArchives { - paths = append(paths, "*.gz") - } - dirFrame, errs := data.NewFileDirectoryFrame(folder, paths) - frames[fmt.Sprintf("logs-%d", i)] = dirFrame - frameErrors = append(frameErrors, errs...) - } - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{Errors: frameErrors}, - }, err -} - -func (lc *LogsCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - } -} - -func FindLogFileCandidates() (logFolders map[string][]string, configErrors []error) { - // we need the config to determine the location of the logs - configCandidates := make(map[string]data.ConfigFileFrame) - configFiles, err := FindConfigurationFiles() - logFolders = make(map[string][]string) - if err != nil { - configErrors = append(configErrors, err) - return logFolders, configErrors - } - for _, folder := range configFiles { - configFrame, errs := data.NewConfigFileFrame(folder) - configErrors = append(configErrors, errs...) - configCandidates[filepath.Clean(folder)] = configFrame - } - - for _, config := range configCandidates { - paths, errs := config.FindLogPaths() - for _, path := range paths { - folder := filepath.Dir(path) - filename := filepath.Base(path) - if _, ok := logFolders[folder]; !ok { - logFolders[folder] = []string{} - } - logFolders[folder] = utils.Unique(append(logFolders[folder], filename)) - } - configErrors = append(configErrors, errs...) - } - return logFolders, configErrors -} - -func (lc *LogsCollector) IsDefault() bool { - return true -} - -func (lc LogsCollector) Description() string { - return "Collects the ClickHouse logs directly from the database." -} - -// here we register the collector for use -func init() { - collectors.Register("logs", func() (collectors.Collector, error) { - return &LogsCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go deleted file mode 100644 index 5f0be734445..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package clickhouse_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestLogsConfiguration(t *testing.T) { - t.Run("correct configuration is returned for logs collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := logsCollector.Configuration() - require.Len(t, conf.Params, 2) - // check directory - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, directory.Name(), "directory") - require.Empty(t, directory.Value) - // check collect_archives - require.IsType(t, config.BoolParam{}, conf.Params[1]) - collectArchives, ok := conf.Params[1].(config.BoolParam) - require.True(t, ok) - require.False(t, collectArchives.Required()) - require.Equal(t, collectArchives.Name(), "collect_archives") - require.False(t, collectArchives.Value) - }) -} - -func TestLogsCollect(t *testing.T) { - - logsCollector := clickhouse.NewLogsCollector(&platform.ResourceManager{}) - - t.Run("test default logs collection", func(t *testing.T) { - // we can't rely on a local installation of clickhouse being present for tests - if it is present (and running) - // results maybe variable e.g. we may find a config. For now, we allow flexibility and test only default. - // TODO: we may want to test this within a container - bundle, err := logsCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, bundle) - // we will have some errors if clickhouse is installed or not. If former, permission issues - if latter missing folders. - require.Greater(t, len(bundle.Errors.Errors), 0) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "default") - _, ok := bundle.Frames["default"].(data.DirectoryFileFrame) - require.True(t, ok) - // no guarantees clickhouse is installed so this bundle could have no frames - }) - - t.Run("test logs collection when directory is specified", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log"}) - - }) - - t.Run("test logs collection of archives", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - logsPath := path.Join(cwd, "../../../testdata", "logs", "var", "logs") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("collect_archives", "Collect compressed log archive files", false), - }, - }, - }) - require.Nil(t, err) - checkDirectoryBundle(t, bundle, logsPath, []string{"clickhouse-server.log", "clickhouse-server.err.log", "clickhouse-server.log.gz"}) - }) - - t.Run("test when directory does not exist", func(t *testing.T) { - tmpDir := t.TempDir() - logsPath := path.Join(tmpDir, "random") - bundle, err := logsCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: logsPath, - Param: config.NewParam("directory", "Specify the location of the log files for ClickHouse Server e.g. /var/log/clickhouse-server/", false), - AllowEmpty: true, - }, - }, - }) - // not a fatal error currently - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, fmt.Sprintf("directory %s does not exist", logsPath), bundle.Errors.Errors[0].Error()) - }) -} - -func checkDirectoryBundle(t *testing.T, bundle *data.DiagnosticBundle, logsPath string, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "user_specified") - dirFrame, ok := bundle.Frames["user_specified"].(data.DirectoryFileFrame) - require.True(t, ok) - require.Equal(t, logsPath, dirFrame.Directory) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - fullPaths := make([]string, len(expectedFiles)) - for i, filePath := range expectedFiles { - fullPaths[i] = path.Join(logsPath, filePath) - } - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, fullPaths, file.FilePath()) - i += 1 - } - require.Equal(t, len(fullPaths), i) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/queries.json b/programs/diagnostics/internal/collectors/clickhouse/queries.json deleted file mode 100644 index f5cf4362c9e..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/queries.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "queries": { - "version": [ - { - "statement": "SELECT version()" - } - ], - "databases": [ - { - "statement": "SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT {{.Limit}}" - } - ], - "access": [ - { - "statement": "SHOW ACCESS" - } - ], - "quotas": [ - { - "statement": "SHOW QUOTA" - } - ], - "db_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.databases GROUP BY engine" - } - ], - "table_engines": [ - { - "statement": "SELECT engine, count() \"count\" FROM system.tables WHERE database != 'system' GROUP BY engine" - } - ], - "dictionaries": [ - { - "statement": "SELECT source, type, status, count() \"count\" FROM system.dictionaries GROUP BY source, type, status ORDER BY status DESC, source" - } - ], - "replicated_tables_by_delay": [ - { - "statement": "SELECT database, table, is_leader, is_readonly, absolute_delay, queue_size, inserts_in_queue, merges_in_queue FROM system.replicas ORDER BY absolute_delay DESC LIMIT {{.Limit}}" - } - ], - "replication_queue_by_oldest": [ - { - "statement": "SELECT database, table, replica_name, position, node_name, type, source_replica, parts_to_merge, new_part_name, create_time, required_quorum, is_detach, is_currently_executing, num_tries, last_attempt_time, last_exception, concat( 'time: ', toString(last_postpone_time), ', number: ', toString(num_postponed), ', reason: ', postpone_reason ) postpone FROM system.replication_queue ORDER BY create_time ASC LIMIT {{.Limit}}" - } - ], - "replicated_fetches": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", partition_id, result_part_name, result_part_path, total_size_bytes_compressed, bytes_read_compressed, source_replica_path, source_replica_hostname, source_replica_port, interserver_scheme, to_detached, thread_id FROM system.replicated_fetches" - } - ], - "tables_by_max_partition_count": [ - { - "statement": "SELECT database, table, count() \"partitions\", sum(part_count) \"parts\", max(part_count) \"max_parts_per_partition\" FROM ( SELECT database, table, partition, count() \"part_count\" FROM system.parts WHERE active GROUP BY database, table, partition ) partitions GROUP BY database, table ORDER BY max_parts_per_partition DESC LIMIT {{.Limit}}" - } - ], - "stack_traces": [ - { - "statement": "SELECT '\\n' || arrayStringConcat( arrayMap( x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace) ), '\\n' ) AS trace FROM system.stack_trace" - } - ], - "crash_log": [ - { - "statement": "SELECT event_time, signal, thread_id, query_id, '\\n' || arrayStringConcat(trace_full, '\\n') AS trace, version FROM system.crash_log ORDER BY event_time DESC" - } - ], - "merges": [ - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, result_part_path, source_part_paths, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\", thread_id FROM system.merges", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, round(elapsed, 1) \"elapsed\", round(100 * progress, 1) \"progress\", is_mutation, partition_id, num_parts, formatReadableSize(total_size_bytes_compressed) \"total_size_compressed\", formatReadableSize(bytes_read_uncompressed) \"read_uncompressed\", formatReadableSize(bytes_written_uncompressed) \"written_uncompressed\", columns_written, formatReadableSize(memory_usage) \"memory_usage\" FROM system.merges" - } - ], - "mutations": [ - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do_names, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, mutation_id, command, create_time, parts_to_do, is_done, latest_failed_part, latest_fail_time, latest_fail_reason FROM system.mutations WHERE NOT is_done ORDER BY create_time DESC" - } - ], - "recent_data_parts": [ - { - "statement": "SELECT database, table, engine, partition_id, name, part_type, active, level, disk_name, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC", - "constraint": ">=20.3" - }, - { - "statement": "SELECT database, table, engine, partition_id, name, active, level, path, marks, rows, bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes, marks_bytes, modification_time, remove_time, refcount, is_frozen, min_date, max_date, min_time, max_time, min_block_number, max_block_number FROM system.parts WHERE modification_time > now() - INTERVAL 3 MINUTE ORDER BY modification_time DESC" - } - ], - "detached_parts": [ - { - "statement": "SELECT database, table, partition_id, name, disk, reason, min_block_number, max_block_number, level FROM system.detached_parts" - } - ], - "processes": [ - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents, Settings FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.8" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC", - "constraint": ">=21.3" - }, - { - "statement": "SELECT elapsed, query_id, normalizeQuery(query) AS normalized_query, is_cancelled, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, formatReadableSize(memory_usage) AS \"memory usage\", user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.processes ORDER BY elapsed DESC" - } - ], - "top_queries_by_duration": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY query_duration_ms DESC LIMIT {{.Limit}}" - } - ], - "top_queries_by_memory": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY ORDER BY memory_usage DESC LIMIT {{.Limit}}" - } - ], - "failed_queries": [ - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents, Settings FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.8" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, databases, tables, columns, used_aggregate_functions, used_aggregate_function_combinators, used_database_engines, used_data_type_families, used_dictionaries, used_formats, used_functions, used_storages, used_table_functions, thread_ids, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}", - "constraint": ">=21.3" - }, - { - "statement": "SELECT type, query_start_time, query_duration_ms, query_id, query_kind, is_initial_query, normalizeQuery(query) AS normalized_query, concat( toString(read_rows), ' rows / ', formatReadableSize(read_bytes) ) AS read, concat( toString(written_rows), ' rows / ', formatReadableSize(written_bytes) ) AS written, concat( toString(result_rows), ' rows / ', formatReadableSize(result_bytes) ) AS result, formatReadableSize(memory_usage) AS \"memory usage\", exception, '\\n' || stack_trace AS stack_trace, user, initial_user, multiIf( empty(client_name), http_user_agent, concat( client_name, ' ', toString(client_version_major), '.', toString(client_version_minor), '.', toString(client_version_patch) ) ) AS client, client_hostname, ProfileEvents.Names, ProfileEvents.Values, Settings.Names, Settings.Values FROM system.query_log WHERE type != 'QueryStart' AND event_date >= today() - 1 AND event_time >= now() - INTERVAL 1 DAY AND exception != '' ORDER BY query_start_time DESC LIMIT {{.Limit}}" - } - ] - } -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary.go b/programs/diagnostics/internal/collectors/clickhouse/summary.go deleted file mode 100644 index 0b6dd3aff20..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary.go +++ /dev/null @@ -1,159 +0,0 @@ -package clickhouse - -import ( - "bytes" - _ "embed" - "encoding/json" - "strings" - "text/template" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/Masterminds/semver" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SummaryCollector struct { - resourceManager *platform.ResourceManager -} - -type querySet struct { - Queries map[string][]query `json:"queries"` -} - -type query struct { - Statement string `json:"statement"` - Constraint string `json:"constraint"` -} - -type ParameterTemplate struct { - Limit int64 -} - -//go:embed queries.json -var queryFile []byte - -func NewSummaryCollector(m *platform.ResourceManager) *SummaryCollector { - return &SummaryCollector{ - resourceManager: m, - } -} - -func (sc *SummaryCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - var queries querySet - err = json.Unmarshal(queryFile, &queries) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to read queries from disk") - } - limit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - paramTemplate := ParameterTemplate{ - Limit: limit, - } - frames := make(map[string]data.Frame) - - serverVersion, err := getServerSemVersion(sc) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Unable to read server version") - } - - var frameErrors []error - for queryId, sqlQueries := range queries.Queries { - // we find the first matching query that satisfies the current version. Empty version means ANY version is - // supported - for _, sqlQuery := range sqlQueries { - var queryConstraint *semver.Constraints - if sqlQuery.Constraint != "" { - queryConstraint, err = semver.NewConstraint(sqlQuery.Constraint) - if err != nil { - //we try another one - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse version %s for query %s", sqlQuery.Constraint, queryId)) - continue - } - } - if sqlQuery.Constraint == "" || queryConstraint.Check(serverVersion) { - tmpl, err := template.New(queryId).Parse(sqlQuery.Statement) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to parse query %s", queryId)) - //we try another one - continue - } - buf := new(bytes.Buffer) - err = tmpl.Execute(buf, paramTemplate) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to process query %s template", queryId)) - //we try another one - continue - } - frame, err := sc.resourceManager.DbClient.ExecuteStatement(queryId, buf.String()) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to execute query %s", queryId)) - //we try another one - } else { - frames[queryId] = frame - // only 1 query executed - break - } - } - } - - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func getServerSemVersion(sc *SummaryCollector) (*semver.Version, error) { - serverVersion, err := sc.resourceManager.DbClient.Version() - if err != nil { - return &semver.Version{}, err - } - //drop the build number - it is not a semantic version - versionComponents := strings.Split(serverVersion, ".") - serverVersion = strings.Join(versionComponents[:len(versionComponents)-1], ".") - return semver.NewVersion(serverVersion) -} - -func (sc *SummaryCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 20, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } -} - -func (sc *SummaryCollector) IsDefault() bool { - return true -} - -func (sc *SummaryCollector) Description() string { - return "Collects summary statistics on the database based on a set of known useful queries." -} - -// here we register the collector for use -func init() { - collectors.Register("summary", func() (collectors.Collector, error) { - return &SummaryCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go deleted file mode 100644 index 92945d987ed..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSummaryConfiguration(t *testing.T) { - t.Run("correct configuration is returned for summary collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := summaryCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.IntParam{}, conf.Params[0]) - limit, ok := conf.Params[0].(config.IntParam) - require.True(t, ok) - require.False(t, limit.Required()) - require.Equal(t, limit.Name(), "row_limit") - require.Equal(t, int64(20), limit.Value) - }) -} - -func TestSummaryCollection(t *testing.T) { - - client := test.NewFakeClickhouseClient(make(map[string][]string)) - versionFrame := test.NewFakeDataFrame("version", []string{"version()"}, - [][]interface{}{ - {"22.1.3.7"}, - }, - ) - client.QueryResponses["SELECT version()"] = &versionFrame - databasesFrame := test.NewFakeDataFrame("databases", []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, - [][]interface{}{ - {"tutorial", "Atomic", 2, 2, 2, "1.70 GiB"}, - {"default", "Atomic", 5, 5, 6, "1.08 GiB"}, - {"system", "Atomic", 11, 24, 70, "1.05 GiB"}, - {"INFORMATION_SCHEMA", "Memory", 0, 0, 0, "0.00 B"}, - {"covid19db", "Atomic", 0, 0, 0, "0.00 B"}, - {"information_schema", "Memory", 0, 0, 0, "0.00 B"}}) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 20"] = &databasesFrame - - summaryCollector := clickhouse.NewSummaryCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default summary collection", func(t *testing.T) { - bundle, errs := summaryCollector.Collect(config.Configuration{}) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - client.Reset() - }) - - t.Run("test summary collection with limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Limit rows on supported queries.", false), - }, - }, - } - bundle, errs := summaryCollector.Collect(conf) - - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 31) - require.NotNil(t, bundle) - // databases will be absent due to limit - require.Len(t, bundle.Frames, 1) - // check version frame - require.Contains(t, bundle.Frames, "version") - require.Equal(t, []string{"version()"}, bundle.Frames["version"].Columns()) - checkFrame(t, bundle.Frames["version"], versionFrame.Rows) - - client.QueryResponses["SELECT name, engine, tables, partitions, parts, formatReadableSize(bytes_on_disk) \"disk_size\" "+ - "FROM system.databases db LEFT JOIN ( SELECT database, uniq(table) \"tables\", uniq(table, partition) \"partitions\", "+ - "count() AS parts, sum(bytes_on_disk) \"bytes_on_disk\" FROM system.parts WHERE active GROUP BY database ) AS db_stats "+ - "ON db.name = db_stats.database ORDER BY bytes_on_disk DESC LIMIT 1"] = &databasesFrame - bundle, errs = summaryCollector.Collect(conf) - require.Empty(t, errs) - require.Len(t, bundle.Errors.Errors, 30) - require.NotNil(t, bundle) - require.Len(t, bundle.Frames, 2) - require.Contains(t, bundle.Frames, "version") - //check databases frame - require.Contains(t, bundle.Frames, "databases") - require.Equal(t, []string{"name", "engine", "tables", "partitions", "parts", "disk_size"}, bundle.Frames["databases"].Columns()) - // this will parse as our mock client does not read statement (specifically the limit clause) when called with execute - checkFrame(t, bundle.Frames["databases"], databasesFrame.Rows) - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system.go b/programs/diagnostics/internal/collectors/clickhouse/system.go deleted file mode 100644 index d47cfd924f3..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system.go +++ /dev/null @@ -1,165 +0,0 @@ -package clickhouse - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -// This collector collects the system db from database - -type SystemDatabaseCollector struct { - resourceManager *platform.ResourceManager -} - -const SystemDatabase = "system" - -// ExcludeColumns columns if we need - this will be refined over time [table_name][columnA, columnB] -var ExcludeColumns = map[string][]string{} - -// BannedTables - Hardcoded list. These are always excluded even if the user doesn't specify in exclude_tables. -//Attempts to export will work but we will warn -var BannedTables = []string{"numbers", "zeros"} - -// OrderBy contains a map of tables to an order by clause - by default we don't order table dumps -var OrderBy = map[string]data.OrderBy{ - "errors": { - Column: "last_error_message", - Order: data.Desc, - }, - "replication_queue": { - Column: "create_time", - Order: data.Asc, - }, -} - -func NewSystemDatabaseCollector(m *platform.ResourceManager) *SystemDatabaseCollector { - return &SystemDatabaseCollector{ - resourceManager: m, - } -} - -func (sc *SystemDatabaseCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - includeTables, err := config.ReadStringListValues(conf, "include_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - excludeTables = checkBannedTables(includeTables, excludeTables) - ds, err := sc.readSystemAllTables(includeTables, excludeTables, rowLimit) - if err != nil { - return &data.DiagnosticBundle{}, err - } - return ds, nil -} - -// all banned tables are added to excluded if not present and not specified in included. Returns new exclude_tables list. -func checkBannedTables(includeTables []string, excludeTables []string) []string { - for _, bannedTable := range BannedTables { - //if its specified we don't add to our exclude list - explicitly included tables take precedence - if !utils.Contains(includeTables, bannedTable) && !utils.Contains(excludeTables, bannedTable) { - excludeTables = append(excludeTables, bannedTable) - } - } - return excludeTables -} - -func (sc *SystemDatabaseCollector) readSystemAllTables(include []string, exclude []string, limit int64) (*data.DiagnosticBundle, error) { - tableNames, err := sc.resourceManager.DbClient.ReadTableNamesForDatabase(SystemDatabase) - if err != nil { - return nil, err - } - var frameErrors []error - if include != nil { - // nil means include everything - tableNames = utils.Intersection(tableNames, include) - if len(tableNames) != len(include) { - // we warn that some included tables aren't present in db - frameErrors = append(frameErrors, fmt.Errorf("some tables specified in the include_tables are not in the system database and will not be exported: %v", - utils.Distinct(include, tableNames))) - } - } - - // exclude tables unless specified in includes - excludedTables := utils.Distinct(exclude, include) - tableNames = utils.Distinct(tableNames, excludedTables) - frames := make(map[string]data.Frame) - - for _, tableName := range tableNames { - var excludeColumns []string - if _, ok := ExcludeColumns[tableName]; ok { - excludeColumns = ExcludeColumns[tableName] - } - orderBy := data.OrderBy{} - if _, ok := OrderBy[tableName]; ok { - orderBy = OrderBy[tableName] - } - frame, err := sc.resourceManager.DbClient.ReadTable(SystemDatabase, tableName, excludeColumns, orderBy, limit) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to collect %s", tableName)) - } else { - frames[tableName] = frame - } - } - - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -func (sc *SystemDatabaseCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect. Takes precedence over exclude_tables. If not specified (default) all tables except exclude_tables.", false), - }, - config.StringListParam{ - Values: []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, - Param: config.NewParam("exclude_tables", "Specify list of tables to not collect.", false), - }, - config.IntParam{ - Value: 100000, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } -} - -func (sc *SystemDatabaseCollector) IsDefault() bool { - return true -} - -func (sc *SystemDatabaseCollector) Description() string { - return "Collects all tables in the system database, except those which have been excluded." -} - -// here we register the collector for use -func init() { - collectors.Register("system_db", func() (collectors.Collector, error) { - return &SystemDatabaseCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/system_test.go b/programs/diagnostics/internal/collectors/clickhouse/system_test.go deleted file mode 100644 index d1b9a6e7859..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/system_test.go +++ /dev/null @@ -1,366 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system db collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := systemDbCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringListParam{}, conf.Params[0]) - includeTables, ok := conf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, includeTables.Required()) - require.Equal(t, includeTables.Name(), "include_tables") - require.Nil(t, includeTables.Values) - // check second param - require.IsType(t, config.StringListParam{}, conf.Params[1]) - excludeTables, ok := conf.Params[1].(config.StringListParam) - require.True(t, ok) - require.False(t, excludeTables.Required()) - require.Equal(t, "exclude_tables", excludeTables.Name()) - require.Equal(t, []string{"licenses", "distributed_ddl_queue", "query_thread_log", "query_log", "asynchronous_metric_log", "zookeeper", "aggregate_function_combinators", "collations", "contributors", "data_type_families", "formats", "graphite_retentions", "numbers", "numbers_mt", "one", "parts_columns", "projection_parts", "projection_parts_columns", "table_engines", "time_zones", "zeros", "zeros_mt"}, excludeTables.Values) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(100000), rowLimit.Value) - }) -} - -func TestSystemDbCollect(t *testing.T) { - - diskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, - ) - clusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - userFrame := test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, - ) - - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"disks", "clusters", "users"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT * FROM system.disks LIMIT 100000"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 100000"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 100000"] = &userFrame - systemDbCollector := clickhouse.NewSystemDatabaseCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default system db collection", func(t *testing.T) { - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test when we pass an includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - // test excludes - t.Run("test when we pass an excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - // test includes which isn't in the list - t.Run("test when we pass an invalid includes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "invalid"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 1) - require.Equal(t, diagSet.Errors.Error(), "some tables specified in the include_tables are not in the "+ - "system database and will not be exported: [invalid]") - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - client.Reset() - }) - - t.Run("test when we use a table with excluded fields", func(t *testing.T) { - excludeDefault := clickhouse.ExcludeColumns - client.QueryResponses["SELECT * EXCEPT(keep_free_space,type) FROM system.disks LIMIT 100000"] = &diskFrame - clickhouse.ExcludeColumns = map[string][]string{ - "disks": {"keep_free_space", "type"}, - } - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("include_tables", "Exclusion", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, []string{"name", "path", "free_space", "total_space"}, diagSet.Frames["disks"].Columns()) - eDiskFrame := test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352}, - }, - ) - checkFrame(t, diagSet.Frames["disks"], eDiskFrame.Rows) - clickhouse.ExcludeColumns = excludeDefault - client.Reset() - }) - - t.Run("test with a low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: 1, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.disks LIMIT 1"] = &diskFrame - client.QueryResponses["SELECT * FROM system.clusters LIMIT 1"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.users LIMIT 1"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - lClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }) - checkFrame(t, diagSet.Frames["clusters"], lClusterFrame.Rows) - client.Reset() - }) - - t.Run("test with a negative low row limit", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - Value: -23, - Param: config.NewParam("row_limit", "Maximum number of rows to collect from any table. Negative values mean unlimited.", false), - }, - }, - } - client.QueryResponses["SELECT * FROM system.clusters"] = &clusterFrame - client.QueryResponses["SELECT * FROM system.disks"] = &diskFrame - client.QueryResponses["SELECT * FROM system.users"] = &userFrame - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - // disks frame - require.Equal(t, "disks", diagSet.Frames["disks"].Name()) - require.Equal(t, diskFrame.ColumnNames, diagSet.Frames["disks"].Columns()) - checkFrame(t, diagSet.Frames["disks"], diskFrame.Rows) - // clusters frame - require.Equal(t, "clusters", diagSet.Frames["clusters"].Name()) - require.Equal(t, clusterFrame.ColumnNames, diagSet.Frames["clusters"].Columns()) - checkFrame(t, diagSet.Frames["clusters"], clusterFrame.Rows) - // users frame - require.Equal(t, "users", diagSet.Frames["users"].Name()) - require.Equal(t, userFrame.ColumnNames, diagSet.Frames["users"].Columns()) - checkFrame(t, diagSet.Frames["users"], userFrame.Rows) - client.Reset() - }) - - t.Run("test that includes overrides excludes", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks"}, - Param: config.NewParam("exclude_tables", "Excluded", false), - }, - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - client.Reset() - }) - - t.Run("test banned", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 2) - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("test banned unless included", func(t *testing.T) { - bannedDefault := clickhouse.BannedTables - clickhouse.BannedTables = []string{"disks"} - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: []string{"disks", "clusters", "users"}, - Param: config.NewParam("include_tables", "Included", false), - }, - }, - } - diagSet, err := systemDbCollector.Collect(conf) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "users") - require.Contains(t, diagSet.Frames, "clusters") - clickhouse.BannedTables = bannedDefault - client.Reset() - }) - - t.Run("tables are ordered if configured", func(t *testing.T) { - defaultOrderBy := clickhouse.OrderBy - clickhouse.OrderBy = map[string]data.OrderBy{ - "clusters": { - Column: "shard_num", - Order: data.Desc, - }, - } - client.QueryResponses["SELECT * FROM system.clusters ORDER BY shard_num DESC LIMIT 100000"] = &clusterFrame - diagSet, err := systemDbCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 3) - clickhouse.OrderBy = defaultOrderBy - oClusterFrame := test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9000, 1, "default", "", 0, 0, 0}, - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - }, - ) - checkFrame(t, diagSet.Frames["clusters"], oClusterFrame.Rows) - client.Reset() - }) - -} - -func checkFrame(t *testing.T, frame data.Frame, rows [][]interface{}) { - i := 0 - for { - values, ok, err := frame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, rows[i], values) - i += 1 - } - require.Equal(t, i, len(rows)) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go deleted file mode 100644 index 78aefeaa0c1..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go +++ /dev/null @@ -1,153 +0,0 @@ -package clickhouse - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/bmatcuk/doublestar/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -// This collector collects the system zookeeper db - -type ZookeeperCollector struct { - resourceManager *platform.ResourceManager -} - -func NewZookeeperCollector(m *platform.ResourceManager) *ZookeeperCollector { - return &ZookeeperCollector{ - resourceManager: m, - } -} - -func (zkc *ZookeeperCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(zkc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - - pathPattern, err := config.ReadStringValue(conf, "path_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - defaultPattern, _ := zkc.Configuration().GetConfigParam("path_pattern") - if defaultPattern.(config.StringParam).Value != pathPattern { - log.Warn().Msgf("Using non default zookeeper glob pattern [%s] - this can potentially cause high query load", pathPattern) - } - maxDepth, err := config.ReadIntValue(conf, "max_depth") - if err != nil { - return &data.DiagnosticBundle{}, err - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return &data.DiagnosticBundle{}, err - } - // we use doublestar for globs as it provides us with ** but also allows us to identify prefix or base paths - if !doublestar.ValidatePattern(pathPattern) { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "%s is not a valid pattern", pathPattern) - } - base, _ := doublestar.SplitPattern(pathPattern) - frames := make(map[string]data.Frame) - hFrame, frameErrors := zkc.collectSubFrames(base, pathPattern, rowLimit, 0, maxDepth) - fErrors := data.FrameErrors{ - Errors: frameErrors, - } - frames["zookeeper_db"] = hFrame - return &data.DiagnosticBundle{ - Frames: frames, - Errors: fErrors, - }, nil -} - -// recursively iterates over the zookeeper sub tables to a max depth, applying the filter and max rows per table -func (zkc *ZookeeperCollector) collectSubFrames(path, pathPattern string, rowLimit, currentDepth, maxDepth int64) (data.HierarchicalFrame, []error) { - var frameErrors []error - var subFrames []data.HierarchicalFrame - - currentDepth += 1 - if currentDepth == maxDepth { - return data.HierarchicalFrame{}, frameErrors - } - match, err := doublestar.PathMatch(pathPattern, path) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Path match failed for pattern %s with path %s", pathPattern, path)) - return data.HierarchicalFrame{}, frameErrors - } - // we allow a single level to be examined or we never get going - if !match && currentDepth > 1 { - return data.HierarchicalFrame{}, frameErrors - } - frame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT name FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path for sub paths %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - - // this isn't ideal, we add re-execute the query to our collection as this will be consumed by the output lazily - outputFrame, err := zkc.resourceManager.DbClient.ExecuteStatement(path, fmt.Sprintf("SELECT * FROM system.zookeeper WHERE path='%s' LIMIT %d", path, rowLimit)) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read zookeeper table path %s", path)) - return data.HierarchicalFrame{}, frameErrors - } - frameComponents := strings.Split(path, "/") - frameId := frameComponents[len(frameComponents)-1] - - for { - values, ok, err := frame.Next() - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "unable to read frame %s", frame.Name())) - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - if !ok { - return data.NewHierarchicalFrame(frameId, outputFrame, subFrames), frameErrors - } - subName := fmt.Sprintf("%v", values[0]) - subPath := fmt.Sprintf("%s/%s", path, subName) - subFrame, errs := zkc.collectSubFrames(subPath, pathPattern, rowLimit, currentDepth, maxDepth) - if subFrame.Name() != "" { - subFrames = append(subFrames, subFrame) - } - frameErrors = append(frameErrors, errs...) - } -} - -func (zkc *ZookeeperCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "/clickhouse/{task_queue}/**", - Param: config.NewParam("path_pattern", "Glob pattern for zookeeper path matching. Change with caution.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("max_depth", "Max depth for zookeeper navigation.", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Maximum number of rows/sub nodes to collect/expand from any zookeeper leaf. Negative values mean unlimited.", false), - }, - }, - } -} - -func (zkc *ZookeeperCollector) IsDefault() bool { - return false -} - -func (zkc *ZookeeperCollector) Description() string { - return "Collects Zookeeper information available within ClickHouse." -} - -// here we register the collector for use -func init() { - collectors.Register("zookeeper_db", func() (collectors.Collector, error) { - return &ZookeeperCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go deleted file mode 100644 index 3e56f6200f0..00000000000 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package clickhouse_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -func TestZookeeperConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system zookeeper collector", func(t *testing.T) { - client := test.NewFakeClickhouseClient(make(map[string][]string)) - zkCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - conf := zkCollector.Configuration() - require.Len(t, conf.Params, 3) - // check first param - require.IsType(t, config.StringParam{}, conf.Params[0]) - pathPattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, pathPattern.Required()) - require.Equal(t, pathPattern.Name(), "path_pattern") - require.Equal(t, "/clickhouse/{task_queue}/**", pathPattern.Value) - // check second param - require.IsType(t, config.IntParam{}, conf.Params[1]) - maxDepth, ok := conf.Params[1].(config.IntParam) - require.True(t, ok) - require.False(t, maxDepth.Required()) - require.Equal(t, "max_depth", maxDepth.Name()) - require.Equal(t, int64(8), maxDepth.Value) - // check third param - require.IsType(t, config.IntParam{}, conf.Params[2]) - rowLimit, ok := conf.Params[2].(config.IntParam) - require.True(t, ok) - require.False(t, rowLimit.Required()) - require.Equal(t, "row_limit", rowLimit.Name()) - require.Equal(t, int64(10), rowLimit.Value) - }) -} - -func TestZookeeperCollect(t *testing.T) { - level1 := test.NewFakeDataFrame("level_1", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - {"task_queue", "", "4", "4", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "1", "0", "0", "0", "1", "5", "/clickhouse"}, - {"copytasks", "", "525608", "525608", "2022-03-09 13:47:39", "2022-03-09 13:47:39", "0", "7", "0", "0", "0", "7", "526100", "/clickhouse"}, - }, - ) - level2 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{ - {"ddl", "", "5", "5", "2022-02-22 13:30:15", "2022-02-22 13:30:15", "0", "0", "0", "0", "0", "0", "5", "/clickhouse/task_queue"}, - }, - ) - level3 := test.NewFakeDataFrame("level_2", []string{"name", "value", "czxid", "mzxid", "ctime", "mtime", "version", "cversion", "aversion", "ephemeralOwner", "dataLength", "numChildren", "pzxid", "path"}, - [][]interface{}{}, - ) - dbTables := map[string][]string{ - clickhouse.SystemDatabase: {"zookeeper"}, - } - client := test.NewFakeClickhouseClient(dbTables) - - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &level1 - // can't reuse the frame as the first frame will be iterated as part of the recursive zookeeper search performed by the collector - cLevel1 := test.NewFakeDataFrame("level_1", level1.Columns(), level1.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse' LIMIT 10"] = &cLevel1 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &level2 - cLevel2 := test.NewFakeDataFrame("level_2", level2.Columns(), level2.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue' LIMIT 10"] = &cLevel2 - client.QueryResponses["SELECT name FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &level3 - cLevel3 := test.NewFakeDataFrame("level_3", level3.Columns(), level3.Rows) - client.QueryResponses["SELECT * FROM system.zookeeper WHERE path='/clickhouse/task_queue/ddl' LIMIT 10"] = &cLevel3 - - zKCollector := clickhouse.NewZookeeperCollector(&platform.ResourceManager{ - DbClient: client, - }) - - t.Run("test default zookeeper collection", func(t *testing.T) { - diagSet, err := zKCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 1) - require.Contains(t, diagSet.Frames, "zookeeper_db") - require.Equal(t, "clickhouse", diagSet.Frames["zookeeper_db"].Name()) - require.IsType(t, data.HierarchicalFrame{}, diagSet.Frames["zookeeper_db"]) - checkFrame(t, diagSet.Frames["zookeeper_db"], level1.Rows) - require.Equal(t, level1.Columns(), diagSet.Frames["zookeeper_db"].Columns()) - hierarchicalFrame := diagSet.Frames["zookeeper_db"].(data.HierarchicalFrame) - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel2.Rows) - require.Equal(t, cLevel2.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - hierarchicalFrame = hierarchicalFrame.SubFrames[0] - require.Len(t, hierarchicalFrame.SubFrames, 1) - checkFrame(t, hierarchicalFrame.SubFrames[0], cLevel3.Rows) - require.Equal(t, cLevel3.Columns(), hierarchicalFrame.SubFrames[0].Columns()) - }) -} diff --git a/programs/diagnostics/internal/collectors/registry.go b/programs/diagnostics/internal/collectors/registry.go deleted file mode 100644 index 5611f947466..00000000000 --- a/programs/diagnostics/internal/collectors/registry.go +++ /dev/null @@ -1,75 +0,0 @@ -package collectors - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Collector interface { - Collect(config config.Configuration) (*data.DiagnosticBundle, error) - Configuration() config.Configuration - IsDefault() bool - Description() string -} - -// Register can be called from init() on a collector in this package -// It will automatically be added to the Collectors map to be called externally -func Register(name string, collector CollectorFactory) { - if name == "diag_trace" { - // we use this to record errors and warnings - log.Fatal().Msgf("diag_trace is a reserved collector name") - } - // names must be unique - if _, ok := Collectors[name]; ok { - log.Fatal().Msgf("More than 1 collector is trying to register under the name %s. Names must be unique.", name) - } - Collectors[name] = collector -} - -// CollectorFactory lets us use a closure to get instances of the collector struct -type CollectorFactory func() (Collector, error) - -var Collectors = map[string]CollectorFactory{} - -func GetCollectorNames(defaultOnly bool) []string { - // can't pre-allocate as not all maybe default - var collectors []string - for collectorName := range Collectors { - collector, err := GetCollectorByName(collectorName) - if err != nil { - log.Fatal().Err(err) - } - if !defaultOnly || (defaultOnly && collector.IsDefault()) { - collectors = append(collectors, collectorName) - } - } - return collectors -} - -func GetCollectorByName(name string) (Collector, error) { - if collectorFactory, ok := Collectors[name]; ok { - //do something here - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - return collector, nil - } - return nil, fmt.Errorf("%s is not a valid collector name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Collectors { - collector, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "collector %s could not be initialized", name) - } - configurations[name] = collector.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/collectors/registry_test.go b/programs/diagnostics/internal/collectors/registry_test.go deleted file mode 100644 index eccc5f2265d..00000000000 --- a/programs/diagnostics/internal/collectors/registry_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package collectors_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/stretchr/testify/require" -) - -func TestGetCollectorNames(t *testing.T) { - t.Run("can get all collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(false) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs", "file", "command", "zookeeper_db"}, collectorNames) - }) - - t.Run("can get default collector names", func(t *testing.T) { - collectorNames := collectors.GetCollectorNames(true) - require.ElementsMatch(t, []string{"system_db", "config", "summary", "system", "logs", "db_logs"}, collectorNames) - }) -} - -func TestGetCollectorByName(t *testing.T) { - - t.Run("can get collector by name", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("system_db") - require.Nil(t, err) - require.Equal(t, clickhouse.NewSystemDatabaseCollector(platform.GetResourceManager()), collector) - }) - - t.Run("fails on non existing collector", func(t *testing.T) { - collector, err := collectors.GetCollectorByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid collector name", err.Error()) - require.Nil(t, collector) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all collector configurations", func(t *testing.T) { - configs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, configs, 9) - require.Contains(t, configs, "system_db") - require.Contains(t, configs, "config") - require.Contains(t, configs, "summary") - require.Contains(t, configs, "system") - require.Contains(t, configs, "logs") - require.Contains(t, configs, "db_logs") - require.Contains(t, configs, "file") - require.Contains(t, configs, "command") - require.Contains(t, configs, "zookeeper_db") - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command.go b/programs/diagnostics/internal/collectors/system/command.go deleted file mode 100644 index ba4dd1e996c..00000000000 --- a/programs/diagnostics/internal/collectors/system/command.go +++ /dev/null @@ -1,90 +0,0 @@ -package system - -import ( - "bytes" - "os/exec" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/google/shlex" - "github.com/pkg/errors" -) - -// This collector runs a user specified command and collects it to a file - -type CommandCollector struct { - resourceManager *platform.ResourceManager -} - -func NewCommandCollector(m *platform.ResourceManager) *CommandCollector { - return &CommandCollector{ - resourceManager: m, - } -} - -func (c *CommandCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(c.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - command, err := config.ReadStringValue(conf, "command") - if err != nil { - return &data.DiagnosticBundle{}, err - } - var frameErrors []error - // shlex to split the commands and args - cmdArgs, err := shlex.Split(command) - if err != nil || len(cmdArgs) == 0 { - return &data.DiagnosticBundle{}, errors.Wrap(err, "Unable to parse command") - } - cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err = cmd.Run() - var sError string - if err != nil { - frameErrors = append(frameErrors, errors.Wrap(err, "Unable to execute command")) - sError = err.Error() - } - memoryFrame := data.NewMemoryFrame("output", []string{"command", "stdout", "stderr", "error"}, [][]interface{}{ - {command, stdout.String(), stderr.String(), sError}, - }) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "output": memoryFrame, - }, - }, nil -} - -func (c *CommandCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - } -} - -func (c *CommandCollector) IsDefault() bool { - return false -} - -func (c *CommandCollector) Description() string { - return "Allows collection of the output from a user specified command" -} - -// here we register the collector for use -func init() { - collectors.Register("command", func() (collectors.Collector, error) { - return &CommandCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/command_test.go b/programs/diagnostics/internal/collectors/system/command_test.go deleted file mode 100644 index 7de00cdabf4..00000000000 --- a/programs/diagnostics/internal/collectors/system/command_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package system_test - -import ( - "fmt" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestCommandConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - conf := commandCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - command, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, command.Required()) - require.Equal(t, command.Name(), "command") - require.Equal(t, "", command.Value) - }) -} - -func TestCommandCollect(t *testing.T) { - t.Run("test simple command with args", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls -l ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - fmt.Println(values) - require.Len(t, values, 4) - require.Equal(t, "ls -l ../../../testdata", values[0]) - require.Contains(t, values[1], "configs") - require.Contains(t, values[1], "docker") - require.Contains(t, values[1], "log") - require.Equal(t, "", values[2]) - require.Equal(t, "", values[3]) - values, ok, err = memFrame.Next() - require.False(t, ok) - require.Nil(t, err) - require.Nil(t, values) - }) - - t.Run("test empty command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - require.Equal(t, "parameter command is invalid - command cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("test invalid command", func(t *testing.T) { - commandCollector := system.NewCommandCollector(&platform.ResourceManager{}) - bundle, err := commandCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "ls --invalid ../../../testdata", - Param: config.NewParam("command", "Command to execute", true), - AllowEmpty: false, - }, - }, - }) - // commands may error with output - we still capture on stderr - require.Nil(t, err) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "Unable to execute command: exit status 2", bundle.Errors.Errors[0].Error()) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "output") - require.Equal(t, bundle.Frames["output"].Columns(), []string{"command", "stdout", "stderr", "error"}) - memFrame := bundle.Frames["output"].(data.MemoryFrame) - values, ok, err := memFrame.Next() - require.True(t, ok) - require.Nil(t, err) - require.Len(t, values, 4) - require.Equal(t, "ls --invalid ../../../testdata", values[0]) - require.Equal(t, "", values[1]) - // exact values here may vary on platform - require.NotEmpty(t, values[2]) - require.NotEmpty(t, values[3]) - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file.go b/programs/diagnostics/internal/collectors/system/file.go deleted file mode 100644 index cda91636c52..00000000000 --- a/programs/diagnostics/internal/collectors/system/file.go +++ /dev/null @@ -1,100 +0,0 @@ -package system - -import ( - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/yargevad/filepathx" -) - -// This collector collects arbitrary user files - -type FileCollector struct { - resourceManager *platform.ResourceManager -} - -func NewFileCollector(m *platform.ResourceManager) *FileCollector { - return &FileCollector{ - resourceManager: m, - } -} - -func (f *FileCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(f.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - filePattern, err := config.ReadStringValue(conf, "file_pattern") - if err != nil { - return &data.DiagnosticBundle{}, err - } - - var frameErrors []error - // this util package supports recursive file matching e.g. /**/* - matches, err := filepathx.Glob(filePattern) - if err != nil { - return &data.DiagnosticBundle{}, errors.Wrapf(err, "Invalid file_pattern \"%s\"", filePattern) - } - - if len(matches) == 0 { - frameErrors = append(frameErrors, errors.New("0 files match glob pattern")) - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - }, nil - } - - var filePaths []string - for _, match := range matches { - fi, err := os.Stat(match) - if err != nil { - frameErrors = append(frameErrors, errors.Wrapf(err, "Unable to read file %s", match)) - } - if !fi.IsDir() { - log.Debug().Msgf("Collecting file %s", match) - filePaths = append(filePaths, match) - } - } - - frame := data.NewFileFrame("collection", filePaths) - - return &data.DiagnosticBundle{ - Errors: data.FrameErrors{Errors: frameErrors}, - Frames: map[string]data.Frame{ - "collection": frame, - }, - }, nil -} - -func (f *FileCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - } -} - -func (f *FileCollector) IsDefault() bool { - return false -} - -func (f *FileCollector) Description() string { - return "Allows collection of user specified files" -} - -// here we register the collector for use -func init() { - collectors.Register("file", func() (collectors.Collector, error) { - return &FileCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/file_test.go b/programs/diagnostics/internal/collectors/system/file_test.go deleted file mode 100644 index 5b1d5b3a92f..00000000000 --- a/programs/diagnostics/internal/collectors/system/file_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestFileConfiguration(t *testing.T) { - t.Run("correct configuration is returned for file collector", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - conf := fileCollector.Configuration() - require.Len(t, conf.Params, 1) - require.IsType(t, config.StringParam{}, conf.Params[0]) - filePattern, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.True(t, filePattern.Required()) - require.Equal(t, filePattern.Name(), "file_pattern") - require.Equal(t, "", filePattern.Value) - }) -} - -func TestFileCollect(t *testing.T) { - - t.Run("test filter patterns work", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.xml", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Errors.Errors) - checkFileBundle(t, bundle, - []string{"../../../testdata/configs/include/xml/server-include.xml", - "../../../testdata/configs/include/xml/user-include.xml", - "../../../testdata/configs/xml/config.xml", - "../../../testdata/configs/xml/users.xml", - "../../../testdata/configs/xml/users.d/default-password.xml", - "../../../testdata/configs/yandex_xml/config.xml", - "../../../testdata/docker/admin.xml", - "../../../testdata/docker/custom.xml"}) - }) - - t.Run("invalid file patterns are detected", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.NotNil(t, err) - require.Equal(t, "parameter file_pattern is invalid - file_pattern cannot be empty", err.Error()) - require.Equal(t, &data.DiagnosticBundle{}, bundle) - }) - - t.Run("check empty matches are reported", func(t *testing.T) { - fileCollector := system.NewFileCollector(&platform.ResourceManager{}) - bundle, err := fileCollector.Collect(config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "../../../testdata/**/*.random", - Param: config.NewParam("file_pattern", "Glob based pattern to specify files for collection", true), - AllowEmpty: false, - }, - }, - }) - require.Nil(t, err) - require.Nil(t, bundle.Frames) - require.Len(t, bundle.Errors.Errors, 1) - require.Equal(t, "0 files match glob pattern", bundle.Errors.Errors[0].Error()) - }) - -} - -func checkFileBundle(t *testing.T, bundle *data.DiagnosticBundle, expectedFiles []string) { - require.NotNil(t, bundle) - require.Nil(t, bundle.Errors.Errors) - require.Len(t, bundle.Frames, 1) - require.Contains(t, bundle.Frames, "collection") - dirFrame, ok := bundle.Frames["collection"].(data.FileFrame) - require.True(t, ok) - require.Equal(t, []string{"files"}, dirFrame.Columns()) - i := 0 - for { - values, ok, err := dirFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - file, ok := values[0].(data.SimpleFile) - require.True(t, ok) - require.Contains(t, expectedFiles, file.FilePath()) - i += 1 - } - require.Equal(t, len(expectedFiles), i) -} diff --git a/programs/diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go deleted file mode 100644 index 69d16f36b8b..00000000000 --- a/programs/diagnostics/internal/collectors/system/system.go +++ /dev/null @@ -1,235 +0,0 @@ -package system - -import ( - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/elastic/gosigar" - "github.com/jaypipes/ghw" - "github.com/matishsiao/goInfo" - "github.com/pkg/errors" -) - -// This collector collects the system overview - -type SystemCollector struct { - resourceManager *platform.ResourceManager -} - -func NewSystemCollector(m *platform.ResourceManager) *SystemCollector { - return &SystemCollector{ - resourceManager: m, - } -} - -func (sc *SystemCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - _, err := conf.ValidateConfig(sc.Configuration()) - if err != nil { - return &data.DiagnosticBundle{}, err - } - frames := make(map[string]data.Frame) - var frameErrors []error - - frameErrors = addStatsToFrame(frames, frameErrors, "disks", getDisk) - frameErrors = addStatsToFrame(frames, frameErrors, "disk_usage", getDiskUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "memory", getMemory) - frameErrors = addStatsToFrame(frames, frameErrors, "memory_usage", getMemoryUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "cpu", getCPU) - //frameErrors = addStatsToFrame(frames, frameErrors, "cpu_usage", getCPUUsage) - - frameErrors = addStatsToFrame(frames, frameErrors, "processes", getProcessList) - - frameErrors = addStatsToFrame(frames, frameErrors, "os", getHostDetails) - - return &data.DiagnosticBundle{ - Frames: frames, - Errors: data.FrameErrors{ - Errors: frameErrors, - }, - }, err -} - -func addStatsToFrame(frames map[string]data.Frame, errors []error, name string, statFunc func() (data.MemoryFrame, error)) []error { - frame, err := statFunc() - if err != nil { - errors = append(errors, err) - } - frames[name] = frame - return errors -} - -func (sc *SystemCollector) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{}, - } -} - -func (sc *SystemCollector) IsDefault() bool { - return true -} - -func getDisk() (data.MemoryFrame, error) { - block, err := ghw.Block() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list block storage") - } - var rows [][]interface{} - columns := []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"} - for _, disk := range block.Disks { - for _, part := range disk.Partitions { - rows = append(rows, []interface{}{disk.Name, disk.SizeBytes, disk.PhysicalBlockSizeBytes, disk.DriveType, disk.StorageController, disk.Vendor, disk.Model, part.Name, part.SizeBytes, part.MountPoint, part.IsReadOnly}) - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getDiskUsage() (data.MemoryFrame, error) { - fsList := gosigar.FileSystemList{} - err := fsList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list filesystems for usage") - } - rows := make([][]interface{}, len(fsList.List)) - columns := []string{"filesystem", "size", "used", "avail", "use%", "mounted on"} - for i, fs := range fsList.List { - dirName := fs.DirName - usage := gosigar.FileSystemUsage{} - err = usage.Get(dirName) - if err == nil { - rows[i] = []interface{}{fs.DevName, usage.Total, usage.Used, usage.Avail, usage.UsePercent(), dirName} - } else { - // we try to output something - rows[i] = []interface{}{fs.DevName, 0, 0, 0, 0, dirName} - } - } - return data.NewMemoryFrame("disk_usage", columns, rows), nil -} - -func getMemory() (data.MemoryFrame, error) { - memory, err := ghw.Memory() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory") - } - columns := []string{"totalPhysical", "totalUsable", "supportedPageSizes"} - rows := make([][]interface{}, 1) - rows[0] = []interface{}{memory.TotalPhysicalBytes, memory.TotalUsableBytes, memory.SupportedPageSizes} - return data.NewMemoryFrame("memory", columns, rows), nil -} - -func getMemoryUsage() (data.MemoryFrame, error) { - mem := gosigar.Mem{} - swap := gosigar.Swap{} - - err := mem.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read memory usage") - } - - err = swap.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to read swap") - } - - columns := []string{"type", "total", "used", "free"} - rows := make([][]interface{}, 3) - - rows[0] = []interface{}{"mem", mem.Total, mem.Used, mem.Free} - rows[1] = []interface{}{"buffers/cache", 0, mem.ActualUsed, mem.ActualFree} - rows[2] = []interface{}{"swap", swap.Total, swap.Used, swap.Free} - return data.NewMemoryFrame("memory_usage", columns, rows), nil - -} - -func getCPU() (data.MemoryFrame, error) { - cpu, err := ghw.CPU() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus") - } - columns := []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"} - var rows [][]interface{} - for _, proc := range cpu.Processors { - for _, core := range proc.Cores { - rows = append(rows, []interface{}{proc.ID, proc.Vendor, proc.Model, core.ID, core.NumThreads, core.LogicalProcessors, strings.Join(proc.Capabilities, " ")}) - } - } - return data.NewMemoryFrame("cpu", columns, rows), nil -} - -// this gets cpu usage vs a listing of arch etc - see getCPU(). This needs successive values as its ticks - not currently used -// see https://github.com/elastic/beats/blob/master/metricbeat/internal/metrics/cpu/metrics.go#L131 for inspiration -//nolint -func getCPUUsage() (data.MemoryFrame, error) { - cpuList := gosigar.CpuList{} - err := cpuList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list cpus for usage") - } - columns := []string{"sys", "nice", "stolen", "irq", "idle", "softIrq", "user", "wait", "total"} - rows := make([][]interface{}, len(cpuList.List), len(cpuList.List)) - for i, cpu := range cpuList.List { - rows[i] = []interface{}{cpu.Sys, cpu.Nice, cpu.Stolen, cpu.Irq, cpu.Idle, cpu.SoftIrq, cpu.User, cpu.Wait, cpu.Total()} - } - return data.NewMemoryFrame("cpu_usage", columns, rows), nil -} - -func getProcessList() (data.MemoryFrame, error) { - pidList := gosigar.ProcList{} - err := pidList.Get() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to list processes") - } - columns := []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"} - rows := make([][]interface{}, len(pidList.List)) - for i, pid := range pidList.List { - state := gosigar.ProcState{} - mem := gosigar.ProcMem{} - time := gosigar.ProcTime{} - args := gosigar.ProcArgs{} - if err := state.Get(pid); err != nil { - continue - } - if err := mem.Get(pid); err != nil { - continue - } - if err := time.Get(pid); err != nil { - continue - } - if err := args.Get(pid); err != nil { - continue - } - rows[i] = []interface{}{pid, state.Ppid, time.FormatStartTime(), time.FormatTotal(), mem.Resident, mem.Size, - mem.PageFaults, mem.MinorFaults, mem.MajorFaults, state.Username, state.State, state.Priority, state.Nice, - strings.Join(args.List, " ")} - } - return data.NewMemoryFrame("process_list", columns, rows), nil -} - -func getHostDetails() (data.MemoryFrame, error) { - gi, err := goInfo.GetInfo() - if err != nil { - return data.MemoryFrame{}, errors.Wrapf(err, "unable to get host summary") - } - columns := []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"} - rows := [][]interface{}{ - {gi.Hostname, gi.OS, gi.GoOS, gi.CPUs, gi.Core, gi.Kernel, gi.Platform}, - } - return data.NewMemoryFrame("os", columns, rows), nil -} - -func (sc *SystemCollector) Description() string { - return "Collects summary OS and hardware statistics for the host" -} - -// here we register the collector for use -func init() { - collectors.Register("system", func() (collectors.Collector, error) { - return &SystemCollector{ - resourceManager: platform.GetResourceManager(), - }, nil - }) -} diff --git a/programs/diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go deleted file mode 100644 index fb1e16bd1ed..00000000000 --- a/programs/diagnostics/internal/collectors/system/system_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package system_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestSystemConfiguration(t *testing.T) { - t.Run("correct configuration is returned for system collector", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - conf := systemCollector.Configuration() - require.Len(t, conf.Params, 0) - require.Equal(t, []config.ConfigParam{}, conf.Params) - }) -} - -func TestSystemCollect(t *testing.T) { - t.Run("test default system collection", func(t *testing.T) { - systemCollector := system.NewSystemCollector(&platform.ResourceManager{}) - diagSet, err := systemCollector.Collect(config.Configuration{}) - require.Nil(t, err) - require.NotNil(t, diagSet) - require.Len(t, diagSet.Errors.Errors, 0) - require.Len(t, diagSet.Frames, 7) - require.Contains(t, diagSet.Frames, "disks") - require.Contains(t, diagSet.Frames, "disk_usage") - require.Contains(t, diagSet.Frames, "memory") - require.Contains(t, diagSet.Frames, "memory_usage") - require.Contains(t, diagSet.Frames, "cpu") - require.Contains(t, diagSet.Frames, "processes") - require.Contains(t, diagSet.Frames, "os") - // responses here will vary depending on platform - mocking seems excessive so we test we have some data - // disks - require.Equal(t, []string{"name", "size", "physicalBlockSize", "driveType", "controller", "vendor", "model", "partitionName", "partitionSize", "mountPoint", "readOnly"}, diagSet.Frames["disks"].Columns()) - diskFrames, err := countFrameRows(diagSet, "disks") - require.Greater(t, diskFrames, 0) - require.Nil(t, err) - // disk usage - require.Equal(t, []string{"filesystem", "size", "used", "avail", "use%", "mounted on"}, diagSet.Frames["disk_usage"].Columns()) - diskUsageFrames, err := countFrameRows(diagSet, "disk_usage") - require.Greater(t, diskUsageFrames, 0) - require.Nil(t, err) - // memory - require.Equal(t, []string{"totalPhysical", "totalUsable", "supportedPageSizes"}, diagSet.Frames["memory"].Columns()) - memoryFrames, err := countFrameRows(diagSet, "memory") - require.Greater(t, memoryFrames, 0) - require.Nil(t, err) - // memory_usage - require.Equal(t, []string{"type", "total", "used", "free"}, diagSet.Frames["memory_usage"].Columns()) - memoryUsageFrames, err := countFrameRows(diagSet, "memory_usage") - require.Greater(t, memoryUsageFrames, 0) - require.Nil(t, err) - // cpu - require.Equal(t, []string{"processor", "vendor", "model", "core", "numThreads", "logical", "capabilities"}, diagSet.Frames["cpu"].Columns()) - cpuFrames, err := countFrameRows(diagSet, "cpu") - require.Greater(t, cpuFrames, 0) - require.Nil(t, err) - // processes - require.Equal(t, []string{"pid", "ppid", "stime", "time", "rss", "size", "faults", "minorFaults", "majorFaults", "user", "state", "priority", "nice", "command"}, diagSet.Frames["processes"].Columns()) - processesFrames, err := countFrameRows(diagSet, "processes") - require.Greater(t, processesFrames, 0) - require.Nil(t, err) - // os - require.Equal(t, []string{"hostname", "os", "goOs", "cpus", "core", "kernel", "platform"}, diagSet.Frames["os"].Columns()) - osFrames, err := countFrameRows(diagSet, "os") - require.Greater(t, osFrames, 0) - require.Nil(t, err) - }) -} - -func countFrameRows(diagSet *data.DiagnosticBundle, frameName string) (int, error) { - frame := diagSet.Frames[frameName] - i := 0 - for { - _, ok, err := frame.Next() - if !ok { - return i, err - } - if err != nil { - return i, err - } - i++ - } -} diff --git a/programs/diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go deleted file mode 100644 index 63847b3addd..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple.go +++ /dev/null @@ -1,344 +0,0 @@ -package file - -import ( - "context" - "encoding/csv" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/mholt/archiver/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -const OutputName = "simple" - -type SubFolderGenerator func() string - -type SimpleOutput struct { - // mainly used for testing to make sub folder deterministic - which it won't be by default as it uses a timestamp - FolderGenerator SubFolderGenerator -} - -func (o SimpleOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(o.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - directory, err := config.ReadStringValue(conf, "directory") - if err != nil { - return data.FrameErrors{}, err - } - directory, err = getWorkingDirectory(directory) - if err != nil { - return data.FrameErrors{}, err - } - subFolder := strconv.FormatInt(utils.MakeTimestamp(), 10) - if o.FolderGenerator != nil { - subFolder = o.FolderGenerator() - } - skipArchive, err := config.ReadBoolValue(conf, "skip_archive") - if err != nil { - return data.FrameErrors{}, err - } - - outputDir := filepath.Join(directory, id, subFolder) - log.Info().Msgf("creating bundle in %s", outputDir) - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - var filePaths []string - for name := range bundles { - bundlePaths, frameError := writeDiagnosticBundle(name, bundles[name], outputDir) - filePaths = append(filePaths, bundlePaths...) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - log.Info().Msg("bundle created") - if !skipArchive { - archiveFilename := filepath.Join(directory, id, fmt.Sprintf("%s.tar.gz", subFolder)) - log.Info().Msgf("compressing bundle to %s", archiveFilename) - // produce a map containing the input paths to the archive paths - we preserve the output directory and hierarchy - archiveMap := createArchiveMap(filePaths, directory) - if err := createArchive(archiveFilename, archiveMap); err != nil { - return frameErrors, err - } - // we delete the original directory leaving just the archive behind - if err := os.RemoveAll(outputDir); err != nil { - return frameErrors, err - } - log.Info().Msgf("archive ready at: %s ", archiveFilename) - } - return frameErrors, nil -} - -func writeDiagnosticBundle(name string, diag *data.DiagnosticBundle, baseDir string) ([]string, data.FrameErrors) { - diagDir := filepath.Join(baseDir, name) - if err := os.MkdirAll(diagDir, os.ModePerm); err != nil { - return nil, data.FrameErrors{Errors: []error{ - errors.Wrapf(err, "unable to create directory for %s", name), - }} - } - frameErrors := data.FrameErrors{} - var filePaths []string - for frameId, frame := range diag.Frames { - fFilePath, errs := writeFrame(frameId, frame, diagDir) - filePaths = append(filePaths, fFilePath...) - if len(errs) > 0 { - // it would be nice if we could wrap this list of errors into something formal but this logs well - frameErrors.Errors = append(frameErrors.Errors, fmt.Errorf("unable to write frame %s for %s", frameId, name)) - frameErrors.Errors = append(frameErrors.Errors, errs...) - } - } - return filePaths, frameErrors -} - -func writeFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - switch f := frame.(type) { - case data.DatabaseFrame: - return writeDatabaseFrame(frameId, f, baseDir) - case data.ConfigFileFrame: - return writeConfigFrame(frameId, f, baseDir) - case data.DirectoryFileFrame: - return processDirectoryFileFrame(frameId, f, baseDir) - case data.FileFrame: - return processFileFrame(frameId, f, baseDir) - case data.HierarchicalFrame: - return writeHierarchicalFrame(frameId, f, baseDir) - default: - // for now our data frame writer supports all frames - return writeDatabaseFrame(frameId, frame, baseDir) - } -} - -func writeHierarchicalFrame(frameId string, frame data.HierarchicalFrame, baseDir string) ([]string, []error) { - filePaths, errs := writeFrame(frameId, frame.DataFrame, baseDir) - for _, subFrame := range frame.SubFrames { - subDir := filepath.Join(baseDir, subFrame.Name()) - if err := os.MkdirAll(subDir, os.ModePerm); err != nil { - errs = append(errs, err) - continue - } - subPaths, subErrs := writeFrame(subFrame.Name(), subFrame, subDir) - filePaths = append(filePaths, subPaths...) - errs = append(errs, subErrs...) - } - return filePaths, errs -} - -func writeDatabaseFrame(frameId string, frame data.Frame, baseDir string) ([]string, []error) { - frameFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.csv", frameId)) - var errs []error - f, err := os.Create(frameFilePath) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - defer f.Close() - w := csv.NewWriter(f) - defer w.Flush() - if err := w.Write(frame.Columns()); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write columns for frame %s", frameId)) - return []string{}, errs - } - // we don't collect an error for every line here like configs and logs - could mean a lot of unnecessary noise - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to read frame %s", frameId)) - return []string{}, errs - } - if !ok { - return []string{frameFilePath}, errs - } - sValues := make([]string, len(values)) - for i, value := range values { - sValues[i] = fmt.Sprintf("%v", value) - } - if err := w.Write(sValues); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to write row for frame %s", frameId)) - return []string{}, errs - } - } -} - -func writeConfigFrame(frameId string, frame data.ConfigFileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - // this holds our files included - includesDirectory := filepath.Join(frameDirectory, "includes") - if err := os.MkdirAll(includesDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create includes directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - configFile := values[0].(data.ConfigFile) - if !configFile.IsIncluded() { - relPath := strings.TrimPrefix(configFile.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } else { - // include files could be anywhere - potentially multiple with the same name. We thus, recreate the directory - // hierarchy under includes to avoid collisions - destPath := path.Join(includesDirectory, configFile.FilePath()) - if err = configFile.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "Unable to copy file %s", configFile.FilePath())) - } - } - - } -} - -func processDirectoryFileFrame(frameId string, frame data.DirectoryFileFrame, baseDir string) ([]string, []error) { - var errs []error - // each set of files goes under its own directory to preserve grouping - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - return []string{frameDirectory}, errs - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - relPath := strings.TrimPrefix(file.FilePath(), frame.Directory) - destPath := path.Join(frameDirectory, relPath) - - if err = file.Copy(destPath, true); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s for frame %s", file, frameId)) - } - } -} - -func processFileFrame(frameId string, frame data.FileFrame, baseDir string) ([]string, []error) { - var errs []error - frameDirectory := filepath.Join(baseDir, frameId) - if err := os.MkdirAll(frameDirectory, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for frame %s", frameId)) - return []string{}, errs - } - for { - values, ok, err := frame.Next() - if err != nil { - errs = append(errs, err) - } - if !ok { - return []string{frameDirectory}, errs - } - file := values[0].(data.SimpleFile) - // we need an absolute path to preserve the directory hierarchy - dir, err := filepath.Abs(filepath.Dir(file.FilePath())) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to determine dir for %s", file.FilePath())) - } - outputDir := filepath.Join(frameDirectory, dir) - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { - errs = append(errs, errors.Wrapf(err, "unable to create directory for %s", file.FilePath())) - } else { - outputPath := filepath.Join(outputDir, filepath.Base(file.FilePath())) - err = file.Copy(outputPath, false) - if err != nil { - errs = append(errs, errors.Wrapf(err, "unable to copy file %s", file.FilePath())) - } - } - } - } -} - -func getWorkingDirectory(path string) (string, error) { - if !filepath.IsAbs(path) { - workingPath, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(workingPath, path), nil - } - return path, nil -} - -func createArchiveMap(filePaths []string, prefix string) map[string]string { - archiveMap := make(map[string]string) - for _, path := range filePaths { - archiveMap[path] = strings.TrimPrefix(path, prefix) - } - return archiveMap -} - -func createArchive(outputFile string, filePaths map[string]string) error { - files, err := archiver.FilesFromDisk(nil, filePaths) - if err != nil { - return err - } - out, err := os.Create(outputFile) - if err != nil { - return err - } - defer out.Close() - format := archiver.CompressedArchive{ - Compression: archiver.Gz{}, - Archival: archiver.Tar{}, - } - err = format.Archive(context.Background(), out, files) - return err -} - -func (o SimpleOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "./", - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - // TODO: add tsv and others here later - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } -} - -func (o SimpleOutput) Description() string { - return "Writes out the diagnostic bundle as files in a structured directory, optionally producing a compressed archive." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return SimpleOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/outputs/file/simple_test.go b/programs/diagnostics/internal/outputs/file/simple_test.go deleted file mode 100644 index 471a1c70cc1..00000000000 --- a/programs/diagnostics/internal/outputs/file/simple_test.go +++ /dev/null @@ -1,468 +0,0 @@ -package file_test - -import ( - "bufio" - "encoding/xml" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" -) - -var clusterFrame = test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, - [][]interface{}{ - {"events", 1, 1, 1, "dalem-local-clickhouse-blue-1", "192.168.144.2", 9000, 1, "default", "", 0, 0, 0}, - {"events", 2, 1, 1, "dalem-local-clickhouse-blue-2", "192.168.144.4", 9001, 1, "default", "", 0, 0, 0}, - {"events", 3, 1, 1, "dalem-local-clickhouse-blue-3", "192.168.144.3", 9002, 1, "default", "", 0, 0, 0}, - }, -) - -var diskFrame = test.NewFakeDataFrame("disks", []string{"name", "path", "free_space", "total_space", "keep_free_space", "type"}, - [][]interface{}{ - {"default", "/var/lib/clickhouse", 1729659346944, 1938213220352, "", "local"}, - }, -) - -var userFrame = test.NewFakeDataFrame("users", []string{"name", "id", "storage", "auth_type", "auth_params", "host_ip", "host_names", "host_names_regexp", "host_names_like"}, - [][]interface{}{ - {"default", "94309d50-4f52-5250-31bd-74fecac179db,users.xml,plaintext_password", "sha256_password", []string{"::0"}, []string{}, []string{}, []string{}}, - }, -) - -func TestConfiguration(t *testing.T) { - t.Run("correct configuration is returned", func(t *testing.T) { - output := file.SimpleOutput{} - conf := output.Configuration() - require.Len(t, conf.Params, 3) - // check first directory param - require.IsType(t, config.StringParam{}, conf.Params[0]) - directory, ok := conf.Params[0].(config.StringParam) - require.True(t, ok) - require.False(t, directory.Required()) - require.Equal(t, "directory", directory.Name()) - require.Equal(t, "./", directory.Value) - // check second format param - require.IsType(t, config.StringOptions{}, conf.Params[1]) - format, ok := conf.Params[1].(config.StringOptions) - require.True(t, ok) - require.False(t, format.Required()) - require.Equal(t, "format", format.Name()) - require.Equal(t, "csv", format.Value) - require.Equal(t, []string{"csv"}, format.Options) - // check third format compress - require.IsType(t, config.BoolParam{}, conf.Params[2]) - skipArchive, ok := conf.Params[2].(config.BoolParam) - require.True(t, ok) - require.False(t, format.Required()) - require.False(t, skipArchive.Value) - }) -} - -func TestWrite(t *testing.T) { - bundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "systemB": { - Frames: map[string]data.Frame{ - "user": userFrame, - }, - }, - } - t.Run("test we can write simple diagnostic sets", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - clusterFile := path.Join(tempDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(tempDir, "test", "test", "systemA", "disk.csv") - userFile := path.Join(tempDir, "test", "test", "systemB", "user.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - require.FileExists(t, userFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - usersLines, err := readFileLines(userFile) - require.Nil(t, err) - require.Len(t, usersLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - resetFrames() - }) - - t.Run("test invalid parameter", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - config.StringOptions{ - Value: "random", - Options: []string{"csv"}, - // TODO: add tsv and others here later - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip compressed archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Equal(t, data.FrameErrors{}, frameErrors) - require.NotNil(t, err) - require.Equal(t, "parameter format is invalid - random is not a valid value for format - [csv]", err.Error()) - resetFrames() - }) - - t.Run("test compression", func(t *testing.T) { - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", bundles, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - archiveFileName := path.Join(tempDir, "test", "test.tar.gz") - fi, err := os.Stat(archiveFileName) - require.Nil(t, err) - require.FileExists(t, archiveFileName) - // compression will vary so lets test range - require.Greater(t, int64(600), fi.Size()) - require.Less(t, int64(200), fi.Size()) - outputFolder := path.Join(tempDir, "test", "test") - // check the folder doesn't exist and is cleaned up - require.NoFileExists(t, outputFolder) - resetFrames() - }) - - t.Run("test support for directory frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - dirFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.log"}) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "logs": dirFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - - // test the usual frames still work - clusterFile := path.Join(destDir, "test", "test", "systemA", "cluster.csv") - diskFile := path.Join(destDir, "test", "test", "systemA", "disk.csv") - require.FileExists(t, clusterFile) - require.FileExists(t, diskFile) - diskLines, err := readFileLines(diskFile) - require.Nil(t, err) - require.Len(t, diskLines, 2) - clusterLines, err := readFileLines(clusterFile) - require.Nil(t, err) - require.Len(t, clusterLines, 4) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), clusterLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", clusterLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", clusterLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", clusterLines[3]) - //test our directory frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, tempDir) - // path here will be //test>/config/logs/ - newPath := path.Join(destDir, "test", "test", "config", "logs", subPath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for config frames", func(t *testing.T) { - xmlConfig := data.XmlConfig{ - XMLName: xml.Name{}, - Clickhouse: data.XmlLoggerConfig{ - XMLName: xml.Name{}, - ErrorLog: "/var/log/clickhouse-server/clickhouse-server.err.log", - Log: "/var/log/clickhouse-server/clickhouse-server.log", - }, - IncludeFrom: "", - } - tempDir := t.TempDir() - confDir := path.Join(tempDir, "conf") - // create an includes file - includesDir := path.Join(tempDir, "includes") - err := os.MkdirAll(includesDir, os.ModePerm) - require.Nil(t, err) - includesPath := path.Join(includesDir, "random.xml") - includesFile, err := os.Create(includesPath) - require.Nil(t, err) - xmlWriter := io.Writer(includesFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - // create 5 temporary config files - files := make([]string, 5) - // set the includes - xmlConfig.IncludeFrom = includesPath - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(confDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.xml", i)) - files[i] = filepath - xmlFile, err := os.Create(filepath) - require.Nil(t, err) - // write a little xml so its valid - xmlWriter := io.Writer(xmlFile) - enc := xml.NewEncoder(xmlWriter) - enc.Indent(" ", " ") - err = enc.Encode(xmlConfig) - require.Nil(t, err) - } - configFrame, errs := data.NewConfigFileFrame(confDir) - require.Empty(t, errs) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "config": { - Frames: map[string]data.Frame{ - "user_specified": configFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - require.Empty(t, frameErrors.Errors) - //test our config frame - for _, filepath := range files { - // check they were copied - subPath := strings.TrimPrefix(filepath, confDir) - // path here will be //test>/config/user_specified/file - newPath := path.Join(destDir, "test", "test", "config", "user_specified", subPath) - require.FileExists(t, newPath) - } - // check our includes file exits - // path here will be //test>/config/user_specified/file/includes - require.FileExists(t, path.Join(destDir, "test", "test", "config", "user_specified", "includes", includesPath)) - resetFrames() - }) - - t.Run("test support for file frames", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - files := createRandomFiles(tempDir, 5) - fileFrame := data.NewFileFrame("collection", files) - fileBundles := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "disk": diskFrame, - "cluster": clusterFrame, - }, - }, - "file": { - Frames: map[string]data.Frame{ - "collection": fileFrame, - }, - }, - } - destDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: destDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - frameErrors, err := output.Write("test", fileBundles, configuration) - require.Nil(t, err) - require.NotNil(t, frameErrors) - //test our directory frame - for _, filepath := range files { - // path here will be //test>/file/collection/ - newPath := path.Join(destDir, "test", "test", "file", "collection", filepath) - require.FileExists(t, newPath) - } - resetFrames() - }) - - t.Run("test support for hierarchical frames", func(t *testing.T) { - bottomFrame := data.NewHierarchicalFrame("bottomLevel", userFrame, []data.HierarchicalFrame{}) - middleFrame := data.NewHierarchicalFrame("middleLevel", diskFrame, []data.HierarchicalFrame{bottomFrame}) - topFrame := data.NewHierarchicalFrame("topLevel", clusterFrame, []data.HierarchicalFrame{middleFrame}) - tempDir := t.TempDir() - configuration := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Param: config.NewParam("directory", "A directory", true), - Value: tempDir, - }, - // turn compression off as the folder will be deleted by default - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Skip archive", false), - }, - }, - } - output := file.SimpleOutput{FolderGenerator: staticFolderName} - hierarchicalBundle := map[string]*data.DiagnosticBundle{ - "systemA": { - Frames: map[string]data.Frame{ - "topLevel": topFrame, - }, - }, - } - frameErrors, err := output.Write("test", hierarchicalBundle, configuration) - require.Nil(t, err) - require.Equal(t, data.FrameErrors{}, frameErrors) - topFile := path.Join(tempDir, "test", "test", "systemA", "topLevel.csv") - middleFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "middleLevel.csv") - bottomFile := path.Join(tempDir, "test", "test", "systemA", "middleLevel", "bottomLevel", "bottomLevel.csv") - require.FileExists(t, topFile) - require.FileExists(t, middleFile) - require.FileExists(t, bottomFile) - topLines, err := readFileLines(topFile) - require.Nil(t, err) - require.Len(t, topLines, 4) - middleLines, err := readFileLines(middleFile) - require.Nil(t, err) - require.Len(t, middleLines, 2) - bottomLines, err := readFileLines(bottomFile) - require.Nil(t, err) - require.Len(t, bottomLines, 2) - require.Equal(t, strings.Join(clusterFrame.ColumnNames, ","), topLines[0]) - require.Equal(t, "events,1,1,1,dalem-local-clickhouse-blue-1,192.168.144.2,9000,1,default,,0,0,0", topLines[1]) - require.Equal(t, "events,2,1,1,dalem-local-clickhouse-blue-2,192.168.144.4,9001,1,default,,0,0,0", topLines[2]) - require.Equal(t, "events,3,1,1,dalem-local-clickhouse-blue-3,192.168.144.3,9002,1,default,,0,0,0", topLines[3]) - resetFrames() - }) -} - -func createRandomFiles(tempDir string, num int) []string { - files := make([]string, num) - for i := 0; i < 5; i++ { - // we want to check hierarchies are preserved so create a simple folder for each file - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - os.MkdirAll(fileDir, os.ModePerm) //nolint:errcheck - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.log", i)) - files[i] = filepath - os.Create(filepath) //nolint:errcheck - } - return files -} - -func resetFrames() { - clusterFrame.Reset() - userFrame.Reset() - diskFrame.Reset() -} - -func readFileLines(filename string) ([]string, error) { - file, err := os.Open(filename) - if err != nil { - return nil, err - } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - return lines, scanner.Err() -} - -func staticFolderName() string { - return "test" -} diff --git a/programs/diagnostics/internal/outputs/registry.go b/programs/diagnostics/internal/outputs/registry.go deleted file mode 100644 index 0187cd9105d..00000000000 --- a/programs/diagnostics/internal/outputs/registry.go +++ /dev/null @@ -1,67 +0,0 @@ -package outputs - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type Output interface { - Write(id string, bundles map[string]*data.DiagnosticBundle, config config.Configuration) (data.FrameErrors, error) - Configuration() config.Configuration - Description() string - // TODO: we will need to implement this for the convert function - //Read(config config.Configuration) (data.DiagnosticBundle, error) -} - -// Register can be called from init() on an output in this package -// It will automatically be added to the Outputs map to be called externally -func Register(name string, output OutputFactory) { - // names must be unique - if _, ok := Outputs[name]; ok { - log.Error().Msgf("More than 1 output is trying to register under the name %s. Names must be unique.", name) - } - Outputs[name] = output -} - -// OutputFactory lets us use a closure to get instances of the output struct -type OutputFactory func() (Output, error) - -var Outputs = map[string]OutputFactory{} - -func GetOutputNames() []string { - outputs := make([]string, len(Outputs)) - i := 0 - for k := range Outputs { - outputs[i] = k - i++ - } - return outputs -} - -func GetOutputByName(name string) (Output, error) { - if outputFactory, ok := Outputs[name]; ok { - //do something here - output, err := outputFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - return output, nil - } - return nil, fmt.Errorf("%s is not a valid output name", name) -} - -func BuildConfigurationOptions() (map[string]config.Configuration, error) { - configurations := make(map[string]config.Configuration) - for name, collectorFactory := range Outputs { - output, err := collectorFactory() - if err != nil { - return nil, errors.Wrapf(err, "output %s could not be initialized", name) - } - configurations[name] = output.Configuration() - } - return configurations, nil -} diff --git a/programs/diagnostics/internal/outputs/registry_test.go b/programs/diagnostics/internal/outputs/registry_test.go deleted file mode 100644 index ba8408e5a59..00000000000 --- a/programs/diagnostics/internal/outputs/registry_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package outputs_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/stretchr/testify/require" -) - -func TestGetOutputNames(t *testing.T) { - t.Run("can get all output names", func(t *testing.T) { - outputNames := outputs.GetOutputNames() - require.ElementsMatch(t, []string{"simple", "report"}, outputNames) - }) - -} - -func TestGetOutputByName(t *testing.T) { - - t.Run("can get output by name", func(t *testing.T) { - output, err := outputs.GetOutputByName("simple") - require.Nil(t, err) - require.Equal(t, file.SimpleOutput{}, output) - }) - - t.Run("fails on non existing output", func(t *testing.T) { - output, err := outputs.GetOutputByName("random") - require.NotNil(t, err) - require.Equal(t, "random is not a valid output name", err.Error()) - require.Nil(t, output) - }) -} - -func TestBuildConfigurationOptions(t *testing.T) { - - t.Run("can get all output configurations", func(t *testing.T) { - outputs, err := outputs.BuildConfigurationOptions() - require.Nil(t, err) - require.Len(t, outputs, 2) - require.Contains(t, outputs, "simple") - require.Contains(t, outputs, "report") - }) -} diff --git a/programs/diagnostics/internal/outputs/terminal/report.go b/programs/diagnostics/internal/outputs/terminal/report.go deleted file mode 100644 index 8337f542457..00000000000 --- a/programs/diagnostics/internal/outputs/terminal/report.go +++ /dev/null @@ -1,284 +0,0 @@ -package terminal - -import ( - "bufio" - "fmt" - "os" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" -) - -const OutputName = "report" - -type ReportOutput struct { -} - -func (r ReportOutput) Write(id string, bundles map[string]*data.DiagnosticBundle, conf config.Configuration) (data.FrameErrors, error) { - conf, err := conf.ValidateConfig(r.Configuration()) - if err != nil { - return data.FrameErrors{}, err - } - format, err := config.ReadStringOptionsValue(conf, "format") - if err != nil { - return data.FrameErrors{}, err - } - nonInteractive, err := config.ReadBoolValue(conf, "continue") - if err != nil { - return data.FrameErrors{}, err - } - maxRows, err := config.ReadIntValue(conf, "row_limit") - if err != nil { - return data.FrameErrors{}, err - } - maxColumns, err := config.ReadIntValue(conf, "column_limit") - if err != nil { - return data.FrameErrors{}, err - } - frameErrors := data.FrameErrors{} - for name := range bundles { - frameError := printDiagnosticBundle(name, bundles[name], format, !nonInteractive, int(maxRows), int(maxColumns)) - frameErrors.Errors = append(frameErrors.Errors, frameError.Errors...) - } - return data.FrameErrors{}, nil -} - -func printDiagnosticBundle(name string, diag *data.DiagnosticBundle, format string, interactive bool, maxRows, maxColumns int) data.FrameErrors { - frameErrors := data.FrameErrors{} - for frameId, frame := range diag.Frames { - printFrameHeader(fmt.Sprintf("%s.%s", name, frameId)) - err := printFrame(frame, format, maxRows, maxColumns) - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - if interactive { - err := waitForEnter() - if err != nil { - frameErrors.Errors = append(frameErrors.Errors, err) - } - } - } - return frameErrors -} - -func waitForEnter() error { - fmt.Println("Press the Enter Key to view the next frame report") - for { - consoleReader := bufio.NewReaderSize(os.Stdin, 1) - input, err := consoleReader.ReadByte() - if err != nil { - return errors.New("Unable to read user input") - } - if input == 3 { - //ctl +c - fmt.Println("Exiting...") - os.Exit(0) - } - if input == 10 { - return nil - } - } -} - -func printFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - switch f := frame.(type) { - case data.DatabaseFrame: - return printDatabaseFrame(f, format, maxRows, maxColumns) - case data.ConfigFileFrame: - return printConfigFrame(f, format) - case data.DirectoryFileFrame: - return printDirectoryFileFrame(f, format, maxRows) - case data.HierarchicalFrame: - return printHierarchicalFrame(f, format, maxRows, maxColumns) - default: - // for now our data frame writer supports all frames - return printDatabaseFrame(f, format, maxRows, maxColumns) - } -} - -func createTable(format string) *tablewriter.Table { - table := tablewriter.NewWriter(os.Stdout) - if format == "markdown" { - table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) - table.SetCenterSeparator("|") - } - return table -} - -func printFrameHeader(title string) { - titleTable := tablewriter.NewWriter(os.Stdout) - titleTable.SetHeader([]string{title}) - titleTable.SetAutoWrapText(false) - titleTable.SetAutoFormatHeaders(true) - titleTable.SetHeaderAlignment(tablewriter.ALIGN_CENTER) - titleTable.SetRowSeparator("\n") - titleTable.SetHeaderLine(false) - titleTable.SetBorder(false) - titleTable.SetTablePadding("\t") // pad with tabs - titleTable.SetNoWhiteSpace(true) - titleTable.Render() -} - -func printHierarchicalFrame(frame data.HierarchicalFrame, format string, maxRows, maxColumns int) error { - err := printDatabaseFrame(frame, format, maxRows, maxColumns) - if err != nil { - return err - } - for _, subFrame := range frame.SubFrames { - err = printHierarchicalFrame(subFrame, format, maxRows, maxColumns) - if err != nil { - return err - } - } - return nil -} - -func printDatabaseFrame(frame data.Frame, format string, maxRows, maxColumns int) error { - table := createTable(format) - table.SetAutoWrapText(false) - columns := len(frame.Columns()) - if maxColumns > 0 && maxColumns < columns { - columns = maxColumns - } - table.SetHeader(frame.Columns()[:columns]) - r := 0 - trunColumns := 0 - for { - values, ok, err := frame.Next() - if !ok || r == maxRows { - table.Render() - if trunColumns > 0 { - warning(fmt.Sprintf("Truncated %d columns, more available...", trunColumns)) - } - if r == maxRows { - warning("Truncated rows, more available...") - } - return err - } - if err != nil { - return err - } - columns := len(values) - // -1 means unlimited - if maxColumns > 0 && maxColumns < columns { - trunColumns = columns - maxColumns - columns = maxColumns - } - row := make([]string, columns) - for i, value := range values { - if i == columns { - break - } - row[i] = fmt.Sprintf("%v", value) - } - table.Append(row) - r++ - } -} - -// currently we dump the whole config - useless in parts -func printConfigFrame(frame data.Frame, format string) error { - for { - values, ok, err := frame.Next() - if !ok { - return err - } - if err != nil { - return err - } - configFile := values[0].(data.File) - dat, err := os.ReadFile(configFile.FilePath()) - if err != nil { - return err - } - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{configFile.FilePath()}) - table.Append([]string{string(dat)}) - table.Render() - } -} - -func printDirectoryFileFrame(frame data.Frame, format string, maxRows int) error { - for { - values, ok, err := frame.Next() - if !ok { - - return err - } - if err != nil { - return err - } - path := values[0].(data.SimpleFile) - file, err := os.Open(path.FilePath()) - if err != nil { - // failure on one file causes rest to be ignored in frame...we could improve this - return errors.Wrapf(err, "Unable to read file %s", path.FilePath()) - } - scanner := bufio.NewScanner(file) - i := 0 - // create a table per row - as each will be a file - table := createTable(format) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(false) - table.ClearRows() - table.SetHeader([]string{path.FilePath()}) - for scanner.Scan() { - if i == maxRows { - fmt.Println() - table.Render() - warning("Truncated lines, more available...") - fmt.Print("\n") - break - } - table.Append([]string{scanner.Text()}) - i++ - } - } -} - -// prints a warning -func warning(s string) { - fmt.Printf("\x1b[%dm%v\x1b[0m%s\n", 33, "WARNING: ", s) -} - -func (r ReportOutput) Configuration() config.Configuration { - return config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Value: "default", - Options: []string{"default", "markdown"}, - Param: config.NewParam("format", "Format of tables. Default is terminal friendly.", false), - }, - config.BoolParam{ - Value: false, - Param: config.NewParam("continue", "Print report with no interaction", false), - }, - config.IntParam{ - Value: 10, - Param: config.NewParam("row_limit", "Max Rows to print per frame.", false), - }, - config.IntParam{ - Value: 8, - Param: config.NewParam("column_limit", "Max Columns to print per frame. Negative is unlimited.", false), - }, - }, - } -} - -func (r ReportOutput) Description() string { - return "Writes out the diagnostic bundle to the terminal as a simple report." -} - -// here we register the output for use -func init() { - outputs.Register(OutputName, func() (outputs.Output, error) { - return ReportOutput{}, nil - }) -} diff --git a/programs/diagnostics/internal/platform/config/models.go b/programs/diagnostics/internal/platform/config/models.go deleted file mode 100644 index 6c76b8f149b..00000000000 --- a/programs/diagnostics/internal/platform/config/models.go +++ /dev/null @@ -1,129 +0,0 @@ -package config - -import ( - "fmt" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -type ConfigParam interface { - Name() string - Required() bool - Description() string - validate(defaultConfig ConfigParam) error -} - -type Configuration struct { - Params []ConfigParam -} - -type Param struct { - name string - description string - required bool -} - -func NewParam(name string, description string, required bool) Param { - return Param{ - name: name, - description: description, - required: required, - } -} - -func (bp Param) Name() string { - return bp.name -} - -func (bp Param) Required() bool { - return bp.required -} - -func (bp Param) Description() string { - return bp.description -} - -func (bp Param) validate(defaultConfig ConfigParam) error { - return nil -} - -func (c Configuration) GetConfigParam(paramName string) (ConfigParam, error) { - for _, param := range c.Params { - if param.Name() == paramName { - return param, nil - } - } - return nil, fmt.Errorf("%s does not exist", paramName) -} - -// ValidateConfig finds the intersection of a config c and a default config. Requires all possible params to be in default. -func (c Configuration) ValidateConfig(defaultConfig Configuration) (Configuration, error) { - var finalParams []ConfigParam - for _, defaultParam := range defaultConfig.Params { - setParam, err := c.GetConfigParam(defaultParam.Name()) - if err == nil { - // check the set value is valid - if err := setParam.validate(defaultParam); err != nil { - return Configuration{}, fmt.Errorf("parameter %s is invalid - %s", defaultParam.Name(), err.Error()) - } - finalParams = append(finalParams, setParam) - } else if defaultParam.Required() { - return Configuration{}, fmt.Errorf("missing required parameter %s - %s", defaultParam.Name(), err.Error()) - } else { - finalParams = append(finalParams, defaultParam) - } - } - return Configuration{ - Params: finalParams, - }, nil -} - -type StringParam struct { - Param - Value string - AllowEmpty bool -} - -func (sp StringParam) validate(defaultConfig ConfigParam) error { - dsp := defaultConfig.(StringParam) - if !dsp.AllowEmpty && strings.TrimSpace(sp.Value) == "" { - return fmt.Errorf("%s cannot be empty", sp.Name()) - } - // if the parameter is not required it doesn't matter - return nil -} - -type StringListParam struct { - Param - Values []string -} - -type StringOptions struct { - Param - Options []string - Value string - AllowEmpty bool -} - -func (so StringOptions) validate(defaultConfig ConfigParam) error { - dso := defaultConfig.(StringOptions) - if !dso.AllowEmpty && strings.TrimSpace(so.Value) == "" { - return fmt.Errorf("%s cannot be empty", so.Name()) - } - if !utils.Contains(dso.Options, so.Value) { - return fmt.Errorf("%s is not a valid value for %s - %v", so.Value, so.Name(), so.Options) - } - // if the parameter is not required it doesn't matter - return nil -} - -type IntParam struct { - Param - Value int64 -} - -type BoolParam struct { - Param - Value bool -} diff --git a/programs/diagnostics/internal/platform/config/models_test.go b/programs/diagnostics/internal/platform/config/models_test.go deleted file mode 100644 index 916d20ec28b..00000000000 --- a/programs/diagnostics/internal/platform/config/models_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -var conf = config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - config.StringParam{ - Value: "random", - Param: config.NewParam("paramB", "", true), - }, - config.StringParam{ - Value: "", - AllowEmpty: true, - Param: config.NewParam("paramC", "", false), - }, - config.StringOptions{ - Value: "random", - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", false), - AllowEmpty: true, - }, - }, -} - -func TestGetConfigParam(t *testing.T) { - - t.Run("can find get config param by name", func(t *testing.T) { - paramA, err := conf.GetConfigParam("paramA") - require.Nil(t, err) - require.NotNil(t, paramA) - require.IsType(t, config.StringListParam{}, paramA) - stringListParam, ok := paramA.(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - }) - - t.Run("throws error on missing element", func(t *testing.T) { - paramZ, err := conf.GetConfigParam("paramZ") - require.Nil(t, paramZ) - require.NotNil(t, err) - require.Equal(t, err.Error(), "paramZ does not exist") - }) -} - -func TestValidateConfig(t *testing.T) { - - t.Run("validate adds the default and allows override", func(t *testing.T) { - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, err) - require.NotNil(t, newConf) - require.Len(t, newConf.Params, 4) - // check first param - require.IsType(t, config.StringListParam{}, newConf.Params[0]) - stringListParam, ok := newConf.Params[0].(config.StringListParam) - require.True(t, ok) - require.False(t, stringListParam.Required()) - require.Equal(t, stringListParam.Name(), "paramA") - require.ElementsMatch(t, stringListParam.Values, []string{"some", "values"}) - // check second param - require.IsType(t, config.StringParam{}, newConf.Params[1]) - stringParam, ok := newConf.Params[1].(config.StringParam) - require.True(t, ok) - require.True(t, stringParam.Required()) - require.Equal(t, "paramB", stringParam.Name()) - require.Equal(t, "custom", stringParam.Value) - }) - - t.Run("validate errors if missing param", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - Values: []string{"some", "values"}, - Param: config.NewParam("paramA", "", false), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "missing required parameter paramB - paramB does not exist", err.Error()) - }) - - t.Run("validate errors if invalid string value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramB is invalid - paramB cannot be empty", err.Error()) - }) - - t.Run("allow empty string value if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "", - Param: config.NewParam("paramC", "", true), - }, - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - t.Run("validate errors if invalid string options value", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "not_random", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Value: "custom", - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Param: config.NewParam("paramD", "", true), - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.Nil(t, newConf.Params) - require.NotNil(t, err) - require.Equal(t, "parameter paramD is invalid - custom is not a valid value for paramD - [random very_random very_very_random]", err.Error()) - }) - - t.Run("allow empty string value for StringOptions if specified", func(t *testing.T) { - //missing required paramB - customConf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: "custom", - Param: config.NewParam("paramB", "", true), - }, - config.StringOptions{ - Param: config.Param{}, - // this isn't ideal we need to ensure options are set for this to validate correctly - Options: []string{"random", "very_random", "very_very_random"}, - Value: "", - }, - }, - } - newConf, err := customConf.ValidateConfig(conf) - require.NotNil(t, newConf.Params) - require.Nil(t, err) - }) - - //TODO: Do we need to test if parameters of the same name but wrong type are passed?? -} diff --git a/programs/diagnostics/internal/platform/config/utils.go b/programs/diagnostics/internal/platform/config/utils.go deleted file mode 100644 index 5f84c38d4f4..00000000000 --- a/programs/diagnostics/internal/platform/config/utils.go +++ /dev/null @@ -1,74 +0,0 @@ -package config - -import ( - "fmt" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" -) - -func ReadStringListValues(conf Configuration, paramName string) ([]string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return nil, err - } - value, ok := param.(StringListParam) - if !ok { - value, ok = param.(StringListParam) - if !ok { - return nil, fmt.Errorf("%s must be a list of strings", paramName) - } - } - - return value.Values, nil -} - -func ReadStringValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringParam) - if !ok { - return "", fmt.Errorf("%s must be a list of strings", paramName) - } - return value.Value, nil -} - -func ReadIntValue(conf Configuration, paramName string) (int64, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return 0, err - } - value, ok := param.(IntParam) - if !ok { - return 9, fmt.Errorf("%s must be an unsigned integer", paramName) - } - return value.Value, nil -} - -func ReadBoolValue(conf Configuration, paramName string) (bool, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return false, err - } - value, ok := param.(BoolParam) - if !ok { - return false, fmt.Errorf("%s must be a boolean", paramName) - } - return value.Value, nil -} - -func ReadStringOptionsValue(conf Configuration, paramName string) (string, error) { - param, err := conf.GetConfigParam(paramName) - if err != nil { - return "", err - } - value, ok := param.(StringOptions) - if !ok { - return "", fmt.Errorf("%s must be a string options", paramName) - } - if !utils.Contains(value.Options, value.Value) { - return "", fmt.Errorf("%s is not a valid option in %v for the the parameter %s", value.Value, value.Options, paramName) - } - return value.Value, nil -} diff --git a/programs/diagnostics/internal/platform/config/utils_test.go b/programs/diagnostics/internal/platform/config/utils_test.go deleted file mode 100644 index 9e03e5e69d2..00000000000 --- a/programs/diagnostics/internal/platform/config/utils_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package config_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" -) - -func TestReadStringListValues(t *testing.T) { - - t.Run("can find a string list param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringListParam{ - Values: []string{"licenses", "settings"}, - Param: config.NewParam("exclude_tables", "Specify list of tables not to collect", false), - }, - }, - } - excludeTables, err := config.ReadStringListValues(conf, "exclude_tables") - require.Nil(t, err) - require.Equal(t, []string{"licenses", "settings"}, excludeTables) - }) - -} - -func TestReadStringValue(t *testing.T) { - - t.Run("can find a string param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - directory, err := config.ReadStringValue(conf, "directory") - require.Nil(t, err) - require.Equal(t, "/tmp/dump", directory) - }) - -} - -func TestReadIntValue(t *testing.T) { - t.Run("can find an integer param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.IntParam{ - // nil means include everything - Value: 10000, - Param: config.NewParam("row_limit", "Max Rows to collect", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - rowLimit, err := config.ReadIntValue(conf, "row_limit") - require.Nil(t, err) - require.Equal(t, int64(10000), rowLimit) - }) - -} - -func TestReadBoolValue(t *testing.T) { - t.Run("can find a boolean param", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.BoolParam{ - // nil means include everything - Value: true, - Param: config.NewParam("compress", "Compress data", false), - }, - config.StringListParam{ - // nil means include everything - Values: nil, - Param: config.NewParam("include_tables", "Specify list of tables to collect", false), - }, - config.StringParam{ - Value: "/tmp/dump", - Param: config.NewParam("directory", "Specify a directory", false), - }, - }, - } - - compress, err := config.ReadBoolValue(conf, "compress") - require.Nil(t, err) - require.True(t, compress) - }) -} - -func TestReadStringOptionsValue(t *testing.T) { - t.Run("can find a string value in a list of options", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "csv", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Nil(t, err) - require.Equal(t, "csv", format) - }) - - t.Run("errors on invalid value", func(t *testing.T) { - conf := config.Configuration{ - Params: []config.ConfigParam{ - config.StringOptions{ - Param: config.NewParam("format", "List of formats", false), - Options: []string{"csv", "tsv", "binary", "json", "ndjson"}, - Value: "random", - AllowEmpty: false, - }, - }, - } - format, err := config.ReadStringOptionsValue(conf, "format") - require.Equal(t, "random is not a valid option in [csv tsv binary json ndjson] for the the parameter format", err.Error()) - require.Equal(t, "", format) - }) -} diff --git a/programs/diagnostics/internal/platform/data/bundle.go b/programs/diagnostics/internal/platform/data/bundle.go deleted file mode 100644 index e4eeede659e..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -import ( - "strings" -) - -// DiagnosticBundle contains the results from a Collector -// each frame can represent a table or collection of data files. By allowing multiple frames a single DiagnosticBundle -// can potentially contain many related tables -type DiagnosticBundle struct { - Frames map[string]Frame - // Errors is a property to be set if the Collector has an error. This can be used to indicate a partial collection - // and failed frames - Errors FrameErrors -} - -type FrameErrors struct { - Errors []error -} - -func (fe *FrameErrors) Error() string { - errors := make([]string, len(fe.Errors)) - for i := range errors { - errors[i] = fe.Errors[i].Error() - } - return strings.Join(errors, "\n") -} diff --git a/programs/diagnostics/internal/platform/data/bundle_test.go b/programs/diagnostics/internal/platform/data/bundle_test.go deleted file mode 100644 index ff9cfc2cf56..00000000000 --- a/programs/diagnostics/internal/platform/data/bundle_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestBundleError(t *testing.T) { - - t.Run("can get a bundle error", func(t *testing.T) { - errs := make([]error, 3) - errs[0] = errors.New("Error 1") - errs[1] = errors.New("Error 2") - errs[2] = errors.New("Error 3") - fErrors := data.FrameErrors{ - Errors: errs, - } - require.Equal(t, `Error 1 -Error 2 -Error 3`, fErrors.Error()) - - }) -} diff --git a/programs/diagnostics/internal/platform/data/database.go b/programs/diagnostics/internal/platform/data/database.go deleted file mode 100644 index d49317d8719..00000000000 --- a/programs/diagnostics/internal/platform/data/database.go +++ /dev/null @@ -1,88 +0,0 @@ -package data - -import ( - "database/sql" - "fmt" - "reflect" - "strings" -) - -type DatabaseFrame struct { - name string - ColumnNames []string - rows *sql.Rows - columnTypes []*sql.ColumnType - vars []interface{} -} - -func NewDatabaseFrame(name string, rows *sql.Rows) (DatabaseFrame, error) { - databaseFrame := DatabaseFrame{} - columnTypes, err := rows.ColumnTypes() - if err != nil { - return DatabaseFrame{}, err - } - databaseFrame.columnTypes = columnTypes - databaseFrame.name = name - vars := make([]interface{}, len(columnTypes)) - columnNames := make([]string, len(columnTypes)) - for i := range columnTypes { - value := reflect.Zero(columnTypes[i].ScanType()).Interface() - vars[i] = &value - columnNames[i] = columnTypes[i].Name() - } - databaseFrame.ColumnNames = columnNames - databaseFrame.vars = vars - databaseFrame.rows = rows - return databaseFrame, nil -} - -func (f DatabaseFrame) Next() ([]interface{}, bool, error) { - values := make([]interface{}, len(f.columnTypes)) - for f.rows.Next() { - if err := f.rows.Scan(f.vars...); err != nil { - return nil, false, err - } - for i := range f.columnTypes { - ptr := reflect.ValueOf(f.vars[i]) - values[i] = ptr.Elem().Interface() - } - return values, true, nil //nolint - } - // TODO: raise issue as this seems to always raise an error - //err := f.rows.Err() - f.rows.Close() - return nil, false, nil -} - -func (f DatabaseFrame) Columns() []string { - return f.ColumnNames -} - -func (f DatabaseFrame) Name() string { - return f.name -} - -type Order int - -const ( - Asc Order = 1 - Desc Order = 2 -) - -type OrderBy struct { - Column string - Order Order -} - -func (o OrderBy) String() string { - if strings.TrimSpace(o.Column) == "" { - return "" - } - switch o.Order { - case Asc: - return fmt.Sprintf(" ORDER BY %s ASC", o.Column) - case Desc: - return fmt.Sprintf(" ORDER BY %s DESC", o.Column) - } - return "" -} diff --git a/programs/diagnostics/internal/platform/data/database_test.go b/programs/diagnostics/internal/platform/data/database_test.go deleted file mode 100644 index 57d89e78efc..00000000000 --- a/programs/diagnostics/internal/platform/data/database_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package data_test - -import ( - "database/sql" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/DATA-DOG/go-sqlmock" - "github.com/stretchr/testify/require" -) - -func TestString(t *testing.T) { - t.Run("can order by asc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Asc, - } - require.Equal(t, " ORDER BY created_at ASC", orderBy.String()) - }) - - t.Run("can order by desc", func(t *testing.T) { - orderBy := data.OrderBy{ - Column: "created_at", - Order: data.Desc, - } - require.Equal(t, " ORDER BY created_at DESC", orderBy.String()) - }) - -} - -func TestNextDatabaseFrame(t *testing.T) { - - t.Run("can iterate sql rows", func(t *testing.T) { - rowValues := [][]interface{}{ - {int64(1), "post_1", "hello"}, - {int64(2), "post_2", "world"}, - {int64(3), "post_3", "goodbye"}, - {int64(4), "post_4", "world"}, - } - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - for i := range rowValues { - mockRows.AddRow(rowValues[i][0], rowValues[i][1], rowValues[i][2]) - } - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - values, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 3) - require.ElementsMatch(t, values, rowValues[i]) - i++ - } - require.Equal(t, 4, i) - }) - - t.Run("can iterate empty sql rows", func(t *testing.T) { - mockRows := sqlmock.NewRows([]string{"id", "title", "body"}) - rows := mockRowsToSqlRows(mockRows) - dbFrame, err := data.NewDatabaseFrame("test", rows) - require.ElementsMatch(t, dbFrame.Columns(), []string{"id", "title", "body"}) - require.Nil(t, err) - i := 0 - for { - _, ok, err := dbFrame.Next() - require.Nil(t, err) - if !ok { - break - } - i++ - } - require.Equal(t, 0, i) - }) -} - -func mockRowsToSqlRows(mockRows *sqlmock.Rows) *sql.Rows { - db, mock, _ := sqlmock.New() - mock.ExpectQuery("select").WillReturnRows(mockRows) - rows, _ := db.Query("select") - return rows -} diff --git a/programs/diagnostics/internal/platform/data/field.go b/programs/diagnostics/internal/platform/data/field.go deleted file mode 100644 index 5e80fc1f467..00000000000 --- a/programs/diagnostics/internal/platform/data/field.go +++ /dev/null @@ -1,8 +0,0 @@ -package data - -type Field struct { - // Name of the field - Name string - // A list of fields that must implement FieldType interface - Values []interface{} -} diff --git a/programs/diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go deleted file mode 100644 index 9760b4b6906..00000000000 --- a/programs/diagnostics/internal/platform/data/file.go +++ /dev/null @@ -1,444 +0,0 @@ -package data - -import ( - "bufio" - "encoding/xml" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -type File interface { - Copy(destPath string, removeSensitive bool) error - FilePath() string -} - -type SimpleFile struct { - Path string -} - -// Copy supports removeSensitive for other file types but for a simple file this doesn't do anything -func (s SimpleFile) Copy(destPath string, removeSensitive bool) error { - // simple copy easiest - if err := utils.CopyFile(s.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", s.FilePath()) - } - return nil -} - -func (s SimpleFile) FilePath() string { - return s.Path -} - -func NewFileFrame(name string, filePaths []string) FileFrame { - i := 0 - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - return FileFrame{ - name: name, - i: &i, - files: files, - } -} - -type FileFrame struct { - name string - i *int - files []File -} - -func (f FileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f FileFrame) Columns() []string { - return []string{"files"} -} - -func (f FileFrame) Name() string { - return f.name -} - -// DirectoryFileFrame represents a set of files under a directory -type DirectoryFileFrame struct { - FileFrame - Directory string -} - -func NewFileDirectoryFrame(directory string, exts []string) (DirectoryFileFrame, []error) { - filePaths, errs := utils.ListFilesInDirectory(directory, exts) - files := make([]File, len(filePaths)) - for i, path := range filePaths { - files[i] = SimpleFile{ - Path: path, - } - } - i := 0 - return DirectoryFileFrame{ - Directory: directory, - FileFrame: FileFrame{ - files: files, - i: &i, - }, - }, errs -} - -func (f DirectoryFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f DirectoryFileFrame) Columns() []string { - return []string{"files"} -} - -func (f DirectoryFileFrame) Name() string { - return f.Directory -} - -type ConfigFile interface { - File - FindLogPaths() ([]string, error) - FindIncludedConfig() (ConfigFile, error) - IsIncluded() bool -} - -type ConfigFileFrame struct { - i *int - Directory string - files []ConfigFile -} - -func (f ConfigFileFrame) Next() ([]interface{}, bool, error) { - if len(f.files) == *(f.i) { - return nil, false, nil - } - file := f.files[*f.i] - *f.i++ - value := make([]interface{}, 1) - value[0] = file - return value, true, nil -} - -func (f ConfigFileFrame) Name() string { - return f.Directory -} - -func NewConfigFileFrame(directory string) (ConfigFileFrame, []error) { - files, errs := utils.ListFilesInDirectory(directory, []string{"*.xml", "*.yaml", "*.yml"}) - // we can't predict the length because of include files - var configs []ConfigFile - - for _, path := range files { - var configFile ConfigFile - switch ext := filepath.Ext(path); ext { - case ".xml": - configFile = XmlConfigFile{ - Path: path, - Included: false, - } - case ".yml": - configFile = YamlConfigFile{ - Path: path, - Included: false, - } - case ".yaml": - configFile = YamlConfigFile{ - Path: path, - } - } - if configFile != nil { - configs = append(configs, configFile) - // add any included configs - iConf, err := configFile.FindIncludedConfig() - if err != nil { - errs = append(errs, err) - } else { - if iConf.FilePath() != "" { - configs = append(configs, iConf) - } - } - } - } - i := 0 - - return ConfigFileFrame{ - i: &i, - Directory: directory, - files: configs, - }, errs -} - -func (f ConfigFileFrame) Columns() []string { - return []string{"config"} -} - -func (f ConfigFileFrame) FindLogPaths() (logPaths []string, errors []error) { - for _, configFile := range f.files { - paths, err := configFile.FindLogPaths() - if err != nil { - errors = append(errors, err) - } else { - logPaths = append(logPaths, paths...) - } - } - return logPaths, errors -} - -type XmlConfigFile struct { - Path string - Included bool -} - -// these patterns will be used to remove sensitive content - matches of the pattern will be replaced with the key -var xmlSensitivePatterns = map[string]*regexp.Regexp{ - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), - "Replaced": regexp.MustCompile(`(.*)`), -} - -func (x XmlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(x.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", x.FilePath()) - } - return nil - } - return sensitiveFileCopy(x.FilePath(), destPath, xmlSensitivePatterns) -} - -func (x XmlConfigFile) FilePath() string { - return x.Path -} - -func (x XmlConfigFile) IsIncluded() bool { - return x.Included -} - -type XmlLoggerConfig struct { - XMLName xml.Name `xml:"logger"` - ErrorLog string `xml:"errorlog"` - Log string `xml:"log"` -} - -type YandexXMLConfig struct { - XMLName xml.Name `xml:"yandex"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -type XmlConfig struct { - XMLName xml.Name `xml:"clickhouse"` - Clickhouse XmlLoggerConfig `xml:"logger"` - IncludeFrom string `xml:"include_from"` -} - -func (x XmlConfigFile) UnmarshallConfig() (XmlConfig, error) { - inputFile, err := ioutil.ReadFile(x.Path) - - if err != nil { - return XmlConfig{}, err - } - var cConfig XmlConfig - err = xml.Unmarshal(inputFile, &cConfig) - if err == nil { - return XmlConfig{ - Clickhouse: cConfig.Clickhouse, - IncludeFrom: cConfig.IncludeFrom, - }, nil - } - // attempt to marshall as yandex file - var yConfig YandexXMLConfig - err = xml.Unmarshal(inputFile, &yConfig) - if err != nil { - return XmlConfig{}, err - } - return XmlConfig{ - Clickhouse: yConfig.Clickhouse, - IncludeFrom: yConfig.IncludeFrom, - }, nil -} - -func (x XmlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - config, err := x.UnmarshallConfig() - if err != nil { - return nil, err - } - if config.Clickhouse.Log != "" { - paths = append(paths, config.Clickhouse.Log) - } - if config.Clickhouse.ErrorLog != "" { - paths = append(paths, config.Clickhouse.ErrorLog) - } - - return paths, nil -} - -func (x XmlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if x.Included { - //can't recurse - return XmlConfigFile{}, nil - } - config, err := x.UnmarshallConfig() - if err != nil { - return XmlConfigFile{}, err - } - // we need to convert this - if config.IncludeFrom != "" { - if filepath.IsAbs(config.IncludeFrom) { - return XmlConfigFile{Path: config.IncludeFrom, Included: true}, nil - } - confDir := filepath.Dir(x.FilePath()) - return XmlConfigFile{Path: path.Join(confDir, config.IncludeFrom), Included: true}, nil - } - return XmlConfigFile{}, nil -} - -type YamlConfigFile struct { - Path string - Included bool -} - -var ymlSensitivePatterns = map[string]*regexp.Regexp{ - "password: 'Replaced'": regexp.MustCompile(`password:\s*.*$`), - "password_sha256_hex: 'Replaced'": regexp.MustCompile(`password_sha256_hex:\s*.*$`), - "access_key_id: 'Replaced'": regexp.MustCompile(`access_key_id:\s*.*$`), - "secret_access_key: 'Replaced'": regexp.MustCompile(`secret_access_key:\s*.*$`), - "secret: 'Replaced'": regexp.MustCompile(`secret:\s*.*$`), -} - -func (y YamlConfigFile) Copy(destPath string, removeSensitive bool) error { - if !removeSensitive { - // simple copy easiest - if err := utils.CopyFile(y.FilePath(), destPath); err != nil { - return errors.Wrapf(err, "unable to copy file %s", y.FilePath()) - } - return nil - } - return sensitiveFileCopy(y.FilePath(), destPath, ymlSensitivePatterns) -} - -func (y YamlConfigFile) FilePath() string { - return y.Path -} - -func (y YamlConfigFile) IsIncluded() bool { - return y.Included -} - -type YamlLoggerConfig struct { - Log string - ErrorLog string -} - -type YamlConfig struct { - Logger YamlLoggerConfig - Include_From string -} - -func (y YamlConfigFile) FindLogPaths() ([]string, error) { - var paths []string - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return nil, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return nil, err - } - if config.Logger.Log != "" { - paths = append(paths, config.Logger.Log) - } - if config.Logger.ErrorLog != "" { - paths = append(paths, config.Logger.ErrorLog) - } - return paths, nil -} - -func (y YamlConfigFile) FindIncludedConfig() (ConfigFile, error) { - if y.Included { - //can't recurse - return YamlConfigFile{}, nil - } - inputFile, err := ioutil.ReadFile(y.Path) - if err != nil { - return YamlConfigFile{}, err - } - var config YamlConfig - err = yaml.Unmarshal(inputFile, &config) - if err != nil { - return YamlConfigFile{}, err - } - if config.Include_From != "" { - if filepath.IsAbs(config.Include_From) { - return YamlConfigFile{Path: config.Include_From, Included: true}, nil - } - confDir := filepath.Dir(y.FilePath()) - return YamlConfigFile{Path: path.Join(confDir, config.Include_From), Included: true}, nil - } - return YamlConfigFile{}, nil -} - -func sensitiveFileCopy(sourcePath string, destPath string, patterns map[string]*regexp.Regexp) error { - destDir := filepath.Dir(destPath) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - // currently, we don't unmarshall into a struct - we want to preserve structure and comments. Possibly could - // be handled but for simplicity we do a line parse for now - inputFile, err := os.Open(sourcePath) - - if err != nil { - return err - } - defer inputFile.Close() - outputFile, err := os.Create(destPath) - - if err != nil { - return err - } - defer outputFile.Close() - writer := bufio.NewWriter(outputFile) - scanner := bufio.NewScanner(inputFile) - - for scanner.Scan() { - line := scanner.Text() - for repl, pattern := range patterns { - line = pattern.ReplaceAllString(line, repl) - } - _, err = writer.WriteString(line + "\n") - if err != nil { - return err - } - } - writer.Flush() - return nil -} diff --git a/programs/diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go deleted file mode 100644 index 9e305b1a5da..00000000000 --- a/programs/diagnostics/internal/platform/data/file_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package data_test - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextFileDirectoryFrame(t *testing.T) { - t.Run("can iterate file frame", func(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - filepath := path.Join(fileDir, fmt.Sprintf("random-%d.txt", i)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*.txt"}) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - require.Equal(t, files[i], values[0].(data.SimpleFile).Path) - i += 1 - } - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - fileFrame, errs := data.NewFileDirectoryFrame(tempDir, []string{"*"}) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := fileFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestNewConfigFileFrame(t *testing.T) { - t.Run("can iterate config file frame", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Len(t, values, 1) - filePath := values[0].(data.XmlConfigFile).FilePath() - require.True(t, strings.Contains(filePath, ".xml")) - i += 1 - } - // 5 not 3 due to the includes - require.Equal(t, 5, i) - }) - - t.Run("can iterate file frame when empty", func(t *testing.T) { - // create 5 temporary files - tempDir := t.TempDir() - configFrame, errs := data.NewConfigFileFrame(tempDir) - require.Empty(t, errs) - i := 0 - for { - _, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} - -func TestConfigFileFrameCopy(t *testing.T) { - t.Run("can copy non-sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - newPath := path.Join(tmrDir, filepath.Base(configFile.FilePath())) - err = configFile.Copy(newPath, false) - require.FileExists(t, newPath) - sourceInfo, _ := os.Stat(configFile.FilePath()) - destInfo, _ := os.Stat(newPath) - require.Equal(t, sourceInfo.Size(), destInfo.Size()) - require.Nil(t, err) - } - }) - - t.Run("can copy sensitive xml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.XmlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.xml" || fileName == "default-password.xml" || fileName == "user-include.xml" { - require.True(t, strings.Contains(s, "Replaced") || - strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } else if fileName == "config.xml" { - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.True(t, strings.Contains(s, "Replaced")) - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - require.NotContains(t, s, "REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.xml", "default-password.xml", "user-include.xml", "config.xml", "server-include.xml"}, checkedFiles) - require.Equal(t, 5, i) - }) - - t.Run("can copy sensitive yaml config files", func(t *testing.T) { - tmrDir := t.TempDir() - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - i := 0 - var checkedFiles []string - for { - values, ok, err := configFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.Nil(t, err) - require.True(t, ok) - configFile := values[0].(data.YamlConfigFile) - fileName := filepath.Base(configFile.FilePath()) - newPath := path.Join(tmrDir, fileName) - err = configFile.Copy(newPath, true) - require.FileExists(t, newPath) - require.Nil(t, err) - bytes, err := ioutil.ReadFile(newPath) - require.Nil(t, err) - s := string(bytes) - checkedFiles = append(checkedFiles, fileName) - if fileName == "users.yaml" || fileName == "default-password.yaml" || fileName == "user-include.yaml" { - require.True(t, strings.Contains(s, "password: 'Replaced'") || - strings.Contains(s, "password_sha256_hex: 'Replaced'")) - require.NotContains(t, s, "password: 'REPLACE_ME'") - require.NotContains(t, s, "password_sha256_hex: \"REPLACE_ME\"") - } else if fileName == "config.yaml" { - require.True(t, strings.Contains(s, "access_key_id: 'Replaced'")) - require.True(t, strings.Contains(s, "secret_access_key: 'Replaced'")) - require.True(t, strings.Contains(s, "secret: 'Replaced'")) - require.NotContains(t, s, "access_key_id: 'REPLACE_ME'") - require.NotContains(t, s, "secret_access_key: REPLACE_ME") - require.NotContains(t, s, "secret: REPLACE_ME") - } - i++ - } - require.ElementsMatch(t, []string{"users.yaml", "default-password.yaml", "user-include.yaml", "config.yaml", "server-include.yaml"}, checkedFiles) - require.Equal(t, 5, i) - }) -} - -func TestConfigFileFrameFindLogPaths(t *testing.T) { - t.Run("can find xml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) - - t.Run("can handle empty log paths", func(t *testing.T) { - configFrame, errs := data.NewConfigFileFrame(t.TempDir()) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.Empty(t, paths) - }) - - t.Run("can find yaml log paths", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yaml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} - -// test the legacy format for ClickHouse xml config files with a yandex root tag -func TestYandexConfigFile(t *testing.T) { - t.Run("can find xml log paths with yandex root", func(t *testing.T) { - cwd, err := os.Getwd() - require.Nil(t, err) - configFrame, errs := data.NewConfigFileFrame(path.Join(cwd, "../../../testdata", "configs", "yandex_xml")) - require.Empty(t, errs) - paths, errs := configFrame.FindLogPaths() - require.Empty(t, errs) - require.ElementsMatch(t, []string{"/var/log/clickhouse-server/clickhouse-server.log", - "/var/log/clickhouse-server/clickhouse-server.err.log"}, paths) - }) -} diff --git a/programs/diagnostics/internal/platform/data/frame.go b/programs/diagnostics/internal/platform/data/frame.go deleted file mode 100644 index 65978430109..00000000000 --- a/programs/diagnostics/internal/platform/data/frame.go +++ /dev/null @@ -1,11 +0,0 @@ -package data - -type BaseFrame struct { - Name string -} - -type Frame interface { - Next() ([]interface{}, bool, error) - Columns() []string - Name() string -} diff --git a/programs/diagnostics/internal/platform/data/memory.go b/programs/diagnostics/internal/platform/data/memory.go deleted file mode 100644 index 25da25cf251..00000000000 --- a/programs/diagnostics/internal/platform/data/memory.go +++ /dev/null @@ -1,35 +0,0 @@ -package data - -type MemoryFrame struct { - i *int - ColumnNames []string - Rows [][]interface{} - name string -} - -func NewMemoryFrame(name string, columns []string, rows [][]interface{}) MemoryFrame { - i := 0 - return MemoryFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f MemoryFrame) Next() ([]interface{}, bool, error) { - if f.Rows == nil || len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f MemoryFrame) Columns() []string { - return f.ColumnNames -} - -func (f MemoryFrame) Name() string { - return f.name -} diff --git a/programs/diagnostics/internal/platform/data/memory_test.go b/programs/diagnostics/internal/platform/data/memory_test.go deleted file mode 100644 index fcc02e37d32..00000000000 --- a/programs/diagnostics/internal/platform/data/memory_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package data_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" -) - -func TestNextMemoryFrame(t *testing.T) { - t.Run("can iterate memory frame", func(t *testing.T) { - columns := []string{"Filesystem", "Size", "Used", "Avail", "Use%", "Mounted on"} - rows := [][]interface{}{ - {"sysfs", 0, 0, 0, 0, "/sys"}, - {"proc", 0, 0, 0, 0, "/proc"}, - {"udev", 33357840384, 0, 33357840384, 0, "/dev"}, - {"devpts", 0, 0, 0, 0, "/dev/pts"}, - {"tmpfs", 6682607616, 2228224, 6680379392, 1, "/run"}, - {"/dev/mapper/system-root", 1938213220352, 118136926208, 1721548947456, 7.000000000000001, "/"}, - } - memoryFrame := data.NewMemoryFrame("disks", columns, rows) - i := 0 - for { - values, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - require.ElementsMatch(t, values, rows[i]) - require.Len(t, values, 6) - i += 1 - } - require.Equal(t, 6, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.NewMemoryFrame("test", []string{}, [][]interface{}{}) - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) - - t.Run("can iterate memory frame when empty", func(t *testing.T) { - memoryFrame := data.MemoryFrame{} - i := 0 - for { - _, ok, err := memoryFrame.Next() - require.Nil(t, err) - if !ok { - break - } - } - require.Equal(t, 0, i) - }) -} diff --git a/programs/diagnostics/internal/platform/data/misc.go b/programs/diagnostics/internal/platform/data/misc.go deleted file mode 100644 index a03213c4f46..00000000000 --- a/programs/diagnostics/internal/platform/data/misc.go +++ /dev/null @@ -1,27 +0,0 @@ -package data - -func NewHierarchicalFrame(name string, frame Frame, subFrames []HierarchicalFrame) HierarchicalFrame { - return HierarchicalFrame{ - name: name, - DataFrame: frame, - SubFrames: subFrames, - } -} - -type HierarchicalFrame struct { - name string - DataFrame Frame - SubFrames []HierarchicalFrame -} - -func (hf HierarchicalFrame) Name() string { - return hf.name -} - -func (hf HierarchicalFrame) Columns() []string { - return hf.DataFrame.Columns() -} - -func (hf HierarchicalFrame) Next() ([]interface{}, bool, error) { - return hf.DataFrame.Next() -} diff --git a/programs/diagnostics/internal/platform/database/native.go b/programs/diagnostics/internal/platform/database/native.go deleted file mode 100644 index 45b9af0349e..00000000000 --- a/programs/diagnostics/internal/platform/database/native.go +++ /dev/null @@ -1,95 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "net/url" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - _ "github.com/ClickHouse/clickhouse-go/v2" - "github.com/pkg/errors" -) - -type ClickhouseNativeClient struct { - host string - connection *sql.DB -} - -func NewNativeClient(host string, port uint16, username string, password string) (*ClickhouseNativeClient, error) { - // debug output ?debug=true - connection, err := sql.Open("clickhouse", fmt.Sprintf("clickhouse://%s:%s@%s:%d/", url.QueryEscape(username), url.QueryEscape(password), host, port)) - if err != nil { - return &ClickhouseNativeClient{}, err - } - if err := connection.Ping(); err != nil { - return &ClickhouseNativeClient{}, err - } - return &ClickhouseNativeClient{ - host: host, - connection: connection, - }, nil -} - -func (c *ClickhouseNativeClient) Ping() error { - return c.connection.Ping() -} - -func (c *ClickhouseNativeClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - rows, err := c.connection.Query(fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause)) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(fmt.Sprintf("%s.%s", databaseName, tableName), rows) -} - -func (c *ClickhouseNativeClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - rows, err := c.connection.Query(fmt.Sprintf("SHOW TABLES FROM %s", databaseName)) - if err != nil { - return nil, err - } - defer rows.Close() - var tableNames []string - var name string - for rows.Next() { - if err := rows.Scan(&name); err != nil { - return nil, err - } - tableNames = append(tableNames, name) - } - return tableNames, nil -} - -func (c *ClickhouseNativeClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - rows, err := c.connection.Query(statement) - if err != nil { - return data.DatabaseFrame{}, err - } - return data.NewDatabaseFrame(id, rows) -} - -func (c *ClickhouseNativeClient) Version() (string, error) { - frame, err := c.ExecuteStatement("version", "SELECT version() as version") - if err != nil { - return "", err - } - values, ok, err := frame.Next() - if err != nil { - return "", err - } - if !ok { - return "", errors.New("unable to read ClickHouse version") - } - if len(values) != 1 { - return "", errors.New("unable to read ClickHouse version - no rows returned") - } - return values[0].(string), nil -} diff --git a/programs/diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go deleted file mode 100644 index 7028a4b4800..00000000000 --- a/programs/diagnostics/internal/platform/database/native_test.go +++ /dev/null @@ -1,289 +0,0 @@ -//go:build !no_docker - -package database_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - // create a ClickHouse container - cwd, err := os.Getwd() - if err != nil { - // can't test without current directory - panic(err) - } - - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without container's port - panic(err) - } - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - return clickhouseContainer, p -} - -func getClient(t *testing.T, mappedPort int) *database.ClickhouseNativeClient { - clickhouseClient, err := database.NewNativeClient("localhost", uint16(mappedPort), "", "") - if err != nil { - t.Fatalf("unable to build client : %v", err) - } - return clickhouseClient -} - -func TestReadTableNamesForDatabase(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - t.Run("client can read tables for a database", func(t *testing.T) { - tables, err := clickhouseClient.ReadTableNamesForDatabase("system") - require.Nil(t, err) - require.GreaterOrEqual(t, len(tables), 70) - require.Contains(t, tables, "merge_tree_settings") - }) -} - -func TestReadTable(t *testing.T) { - t.Run("client can get all rows for system.disks table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.disks as this should contain only 1 row - frame, err := clickhouseClient.ReadTable("system", "disks", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [9]string{"name", "path", "free_space", "total_space", "unreserved_space", "keep_free_space", "type", "is_encrypted", "cache_path"}) - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, "default", values[0]) - require.Equal(t, "/var/lib/clickhouse/", values[1]) - require.Greater(t, values[2], uint64(0)) - require.Greater(t, values[3], uint64(0)) - require.Greater(t, values[4], uint64(0)) - require.Equal(t, values[5], uint64(0)) - require.Equal(t, "local", values[6]) - require.Equal(t, values[7], uint8(0)) - require.Equal(t, values[8], "") - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - // we read the table system.databases as this should be small and consistent on fresh db instances - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}} - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can get all rows for system.databases table with except", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{"data_path", "comment"}, data.OrderBy{}, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [4]string{"name", "engine", "metadata_path", "uuid"}) - }) - - t.Run("client can limit rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{}, 1) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [1][3]string{{"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}} - i := 0 - for { - values, ok, err := frame.Next() - if i == 0 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) - - t.Run("client can order rows for system.databases", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - frame, err := clickhouseClient.ReadTable("system", "databases", []string{}, data.OrderBy{ - Column: "engine", - Order: data.Asc, - }, 10) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [6]string{"name", "engine", "data_path", "metadata_path", "uuid", "comment"}) - expectedRows := [4][3]string{ - {"default", "Atomic", "/var/lib/clickhouse/store/"}, - {"system", "Atomic", "/var/lib/clickhouse/store/"}, - {"INFORMATION_SCHEMA", "Memory", "/var/lib/clickhouse/"}, - {"information_schema", "Memory", "/var/lib/clickhouse/"}, - } - i := 0 - for { - values, ok, err := frame.Next() - - if i < 4 { - require.Nil(t, err) - require.True(t, ok) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - require.Equal(t, expectedRows[i][2], values[2]) - require.NotNil(t, values[3]) - require.NotNil(t, values[4]) - require.Equal(t, "", values[5]) - } else { - require.False(t, ok) - break - } - i += 1 - } - }) -} - -func TestExecuteStatement(t *testing.T) { - t.Run("client can execute any statement", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - statement := "SELECT path, count(*) as count FROM system.disks GROUP BY path;" - frame, err := clickhouseClient.ExecuteStatement("engines", statement) - require.Nil(t, err) - require.ElementsMatch(t, frame.Columns(), [2]string{"path", "count"}) - expectedRows := [1][2]interface{}{ - {"/var/lib/clickhouse/", uint64(1)}, - } - i := 0 - for { - values, ok, err := frame.Next() - if !ok { - require.Nil(t, err) - break - } - require.Nil(t, err) - require.Equal(t, expectedRows[i][0], values[0]) - require.Equal(t, expectedRows[i][1], values[1]) - i++ - } - fmt.Println(i) - }) -} - -func TestVersion(t *testing.T) { - t.Run("client can read version", func(t *testing.T) { - ctx := context.Background() - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - clickhouseClient := getClient(t, mappedPort.Int()) - - version, err := clickhouseClient.Version() - require.Nil(t, err) - require.NotEmpty(t, version) - }) -} diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go deleted file mode 100644 index b4435b62ea2..00000000000 --- a/programs/diagnostics/internal/platform/manager.go +++ /dev/null @@ -1,49 +0,0 @@ -package platform - -import ( - "errors" - "sync" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" -) - -var once sync.Once -var dbInit sync.Once - -// manages all resources that collectors and outputs may wish to ensure inc. db connections - -type DBClient interface { - ReadTableNamesForDatabase(databaseName string) ([]string, error) - ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) - ExecuteStatement(id string, statement string) (data.Frame, error) - Version() (string, error) -} - -var manager *ResourceManager - -type ResourceManager struct { - DbClient DBClient -} - -func GetResourceManager() *ResourceManager { - once.Do(func() { - manager = &ResourceManager{} - }) - return manager -} - -func (m *ResourceManager) Connect(host string, port uint16, username string, password string) error { - var err error - var clientInstance DBClient - init := false - dbInit.Do(func() { - clientInstance, err = database.NewNativeClient(host, port, username, password) - manager.DbClient = clientInstance - init = true - }) - if !init { - return errors.New("connect can only be called once") - } - return err -} diff --git a/programs/diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go deleted file mode 100644 index e6c50c6e505..00000000000 --- a/programs/diagnostics/internal/platform/manager_test.go +++ /dev/null @@ -1,100 +0,0 @@ -//go:build !no_docker - -package platform_test - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/docker/go-connections/nat" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// create a ClickHouse container -func createClickHouseContainer(t *testing.T, ctx context.Context) (testcontainers.Container, nat.Port) { - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, err := clickhouseContainer.MappedPort(ctx, "9000") - if err != nil { - // can't test without a port - panic(err) - } - - return clickhouseContainer, p -} - -func TestConnect(t *testing.T) { - t.Run("can only connect once", func(t *testing.T) { - ctx := context.Background() - - clickhouseContainer, mappedPort := createClickHouseContainer(t, ctx) - defer clickhouseContainer.Terminate(ctx) //nolint - - t.Setenv("CLICKHOUSE_DB_PORT", mappedPort.Port()) - - port := mappedPort.Int() - - // get before connection - manager := platform.GetResourceManager() - require.Nil(t, manager.DbClient) - // init connection - err := manager.Connect("localhost", uint16(port), "", "") - require.Nil(t, err) - require.NotNil(t, manager.DbClient) - // try and re-fetch connection - err = manager.Connect("localhost", uint16(port), "", "") - require.NotNil(t, err) - require.Equal(t, "connect can only be called once", err.Error()) - }) - -} - -func TestGetResourceManager(t *testing.T) { - t.Run("get resource manager", func(t *testing.T) { - manager := platform.GetResourceManager() - require.NotNil(t, manager) - manager2 := platform.GetResourceManager() - require.NotNil(t, manager2) - require.Equal(t, &manager, &manager2) - }) - -} diff --git a/programs/diagnostics/internal/platform/test/data.go b/programs/diagnostics/internal/platform/test/data.go deleted file mode 100644 index 7710e9a69a1..00000000000 --- a/programs/diagnostics/internal/platform/test/data.go +++ /dev/null @@ -1,166 +0,0 @@ -package test - -import ( - "fmt" - "sort" - "strings" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/pkg/errors" -) - -type fakeClickhouseClient struct { - tables map[string][]string - QueryResponses map[string]*FakeDataFrame -} - -func NewFakeClickhouseClient(tables map[string][]string) fakeClickhouseClient { - queryResponses := make(map[string]*FakeDataFrame) - return fakeClickhouseClient{ - tables: tables, - QueryResponses: queryResponses, - } -} - -func (f fakeClickhouseClient) ReadTableNamesForDatabase(databaseName string) ([]string, error) { - if _, ok := f.tables[databaseName]; ok { - return f.tables[databaseName], nil - } - return nil, fmt.Errorf("database %s does not exist", databaseName) -} - -func (f fakeClickhouseClient) ReadTable(databaseName string, tableName string, excludeColumns []string, orderBy data.OrderBy, limit int64) (data.Frame, error) { - - exceptClause := "" - if len(excludeColumns) > 0 { - exceptClause = fmt.Sprintf("EXCEPT(%s) ", strings.Join(excludeColumns, ",")) - } - limitClause := "" - if limit >= 0 { - limitClause = fmt.Sprintf(" LIMIT %d", limit) - } - query := fmt.Sprintf("SELECT * %sFROM %s.%s%s%s", exceptClause, databaseName, tableName, orderBy.String(), limitClause) - frame, error := f.ExecuteStatement(fmt.Sprintf("read_table_%s.%s", databaseName, tableName), query) - if error != nil { - return frame, error - } - fFrame := *(frame.(*FakeDataFrame)) - fFrame = fFrame.FilterColumns(excludeColumns) - fFrame = fFrame.Order(orderBy) - fFrame = fFrame.Limit(limit) - return fFrame, nil -} - -func (f fakeClickhouseClient) ExecuteStatement(id string, statement string) (data.Frame, error) { - if frame, ok := f.QueryResponses[statement]; ok { - return frame, nil - } - return FakeDataFrame{}, errors.New(fmt.Sprintf("No recorded response for %s", statement)) -} - -func (f fakeClickhouseClient) Version() (string, error) { - return "21.12.3", nil -} - -func (f fakeClickhouseClient) Reset() { - for key, frame := range f.QueryResponses { - frame.Reset() - f.QueryResponses[key] = frame - } -} - -type FakeDataFrame struct { - i *int - Rows [][]interface{} - ColumnNames []string - name string -} - -func NewFakeDataFrame(name string, columns []string, rows [][]interface{}) FakeDataFrame { - i := 0 - return FakeDataFrame{ - i: &i, - Rows: rows, - ColumnNames: columns, - name: name, - } -} - -func (f FakeDataFrame) Next() ([]interface{}, bool, error) { - if len(f.Rows) == *(f.i) { - return nil, false, nil - } - value := f.Rows[*f.i] - *f.i++ - return value, true, nil -} - -func (f FakeDataFrame) Columns() []string { - return f.ColumnNames -} - -func (f FakeDataFrame) Name() string { - return f.name -} - -func (f *FakeDataFrame) Reset() { - i := 0 - f.i = &i -} - -func (f FakeDataFrame) FilterColumns(excludeColumns []string) FakeDataFrame { - // get columns we can remove - rColumns := utils.Intersection(f.ColumnNames, excludeColumns) - rIndexes := make([]int, len(rColumns)) - // find the indexes of the columns to remove - for i, column := range rColumns { - rIndexes[i] = utils.IndexOf(f.ColumnNames, column) - } - newRows := make([][]interface{}, len(f.Rows)) - for r, row := range f.Rows { - newRow := row - for i, index := range rIndexes { - newRow = utils.Remove(newRow, index-i) - } - newRows[r] = newRow - } - f.Rows = newRows - f.ColumnNames = utils.Distinct(f.ColumnNames, excludeColumns) - return f -} - -func (f FakeDataFrame) Limit(rowLimit int64) FakeDataFrame { - if rowLimit >= 0 { - if int64(len(f.Rows)) > rowLimit { - f.Rows = f.Rows[:rowLimit] - } - } - return f -} - -func (f FakeDataFrame) Order(orderBy data.OrderBy) FakeDataFrame { - if orderBy.Column == "" { - return f - } - cIndex := utils.IndexOf(f.ColumnNames, orderBy.Column) - sort.Slice(f.Rows, func(i, j int) bool { - left := f.Rows[i][cIndex] - right := f.Rows[j][cIndex] - if iLeft, ok := left.(int); ok { - if orderBy.Order == data.Asc { - return iLeft < right.(int) - } - return iLeft > right.(int) - } else { - // we aren't a full db - revert to string order - sLeft := left.(string) - sRight := right.(string) - if orderBy.Order == data.Asc { - return sLeft < sRight - } - return sLeft > sRight - } - }) - return f -} diff --git a/programs/diagnostics/internal/platform/test/env.go b/programs/diagnostics/internal/platform/test/env.go deleted file mode 100644 index 36b03772ab0..00000000000 --- a/programs/diagnostics/internal/platform/test/env.go +++ /dev/null @@ -1,16 +0,0 @@ -package test - -import "os" - -const defaultClickHouseVersion = "latest" - -func GetClickHouseTestVersion() string { - return GetEnv("CLICKHOUSE_VERSION", defaultClickHouseVersion) -} - -func GetEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} diff --git a/programs/diagnostics/internal/platform/utils/file.go b/programs/diagnostics/internal/platform/utils/file.go deleted file mode 100644 index 71af4b32658..00000000000 --- a/programs/diagnostics/internal/platform/utils/file.go +++ /dev/null @@ -1,95 +0,0 @@ -package utils - -import ( - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -func FileExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if !f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a directory", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func DirExists(name string) (bool, error) { - f, err := os.Stat(name) - if err == nil { - if f.IsDir() { - return true, nil - } - return false, fmt.Errorf("%s is a file", name) - } - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, err -} - -func CopyFile(sourceFilename string, destFilename string) error { - exists, err := FileExists(sourceFilename) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("%s does not exist", sourceFilename) - } - source, err := os.Open(sourceFilename) - if err != nil { - return err - } - defer source.Close() - destDir := filepath.Dir(destFilename) - if err := os.MkdirAll(destDir, os.ModePerm); err != nil { - return errors.Wrapf(err, "unable to create directory %s", destDir) - } - - destination, err := os.Create(destFilename) - if err != nil { - return err - } - defer destination.Close() - _, err = io.Copy(destination, source) - return err -} - -// patterns passed are an OR - any can be satisfied and the file will be listed - -func ListFilesInDirectory(directory string, patterns []string) ([]string, []error) { - var files []string - exists, err := DirExists(directory) - if err != nil { - return files, []error{err} - } - if !exists { - return files, []error{fmt.Errorf("directory %s does not exist", directory)} - } - var pathErrors []error - _ = filepath.Walk(directory, func(path string, info fs.FileInfo, err error) error { - if err != nil { - pathErrors = append(pathErrors, err) - } else if !info.IsDir() { - for _, pattern := range patterns { - if matched, err := filepath.Match(pattern, filepath.Base(path)); err != nil { - pathErrors = append(pathErrors, err) - } else if matched { - files = append(files, path) - } - } - } - return nil - }) - return files, pathErrors -} diff --git a/programs/diagnostics/internal/platform/utils/file_test.go b/programs/diagnostics/internal/platform/utils/file_test.go deleted file mode 100644 index 8d0430090c9..00000000000 --- a/programs/diagnostics/internal/platform/utils/file_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package utils_test - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestFileExists(t *testing.T) { - t.Run("returns true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.FileExists(filepath) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for not existence file", func(t *testing.T) { - tempDir := t.TempDir() - file := path.Join(tempDir, "random.txt") - exists, err := utils.FileExists(file) - require.False(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.FileExists(tempDir) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a directory", tempDir), err.Error()) - }) -} - -func TestDirExists(t *testing.T) { - t.Run("doesn't return true for file", func(t *testing.T) { - tempDir := t.TempDir() - filepath := path.Join(tempDir, "random.txt") - _, err := os.Create(filepath) - require.Nil(t, err) - exists, err := utils.DirExists(filepath) - require.False(t, exists) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s is a file", filepath), err.Error()) - }) - - t.Run("returns true for directory", func(t *testing.T) { - tempDir := t.TempDir() - exists, err := utils.DirExists(tempDir) - require.True(t, exists) - require.Nil(t, err) - }) - - t.Run("doesn't return true random directory", func(t *testing.T) { - exists, err := utils.FileExists(fmt.Sprintf("%d", utils.MakeTimestamp())) - require.False(t, exists) - require.Nil(t, err) - }) -} - -func TestCopyFile(t *testing.T) { - t.Run("can copy file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("can copy nested file", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - _, err := os.Create(sourcePath) - require.Nil(t, err) - destPath := path.Join(tempDir, "sub_dir", "random-2.txt") - err = utils.CopyFile(sourcePath, destPath) - require.Nil(t, err) - }) - - t.Run("fails when file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - sourcePath := path.Join(tempDir, "random.txt") - destPath := path.Join(tempDir, "random-2.txt") - err := utils.CopyFile(sourcePath, destPath) - require.NotNil(t, err) - require.Equal(t, fmt.Sprintf("%s does not exist", sourcePath), err.Error()) - }) -} - -func TestListFilesInDirectory(t *testing.T) { - tempDir := t.TempDir() - files := make([]string, 5) - for i := 0; i < 5; i++ { - fileDir := path.Join(tempDir, fmt.Sprintf("%d", i)) - err := os.MkdirAll(fileDir, os.ModePerm) - require.Nil(t, err) - ext := ".txt" - if i%2 == 0 { - ext = ".csv" - } - filepath := path.Join(fileDir, fmt.Sprintf("random-%d%s", i, ext)) - files[i] = filepath - _, err = os.Create(filepath) - require.Nil(t, err) - } - - t.Run("can list all files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - - t.Run("can list by extension", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv"}) - require.Len(t, mFiles, 3) - require.Empty(t, errs) - require.ElementsMatch(t, []string{files[0], files[2], files[4]}, mFiles) - }) - - t.Run("can list on multiple extensions files", func(t *testing.T) { - mFiles, errs := utils.ListFilesInDirectory(tempDir, []string{"*.csv", "*.txt"}) - require.Len(t, mFiles, 5) - require.Empty(t, errs) - }) - -} diff --git a/programs/diagnostics/internal/platform/utils/process.go b/programs/diagnostics/internal/platform/utils/process.go deleted file mode 100644 index 7b27c215eea..00000000000 --- a/programs/diagnostics/internal/platform/utils/process.go +++ /dev/null @@ -1,49 +0,0 @@ -package utils - -import ( - "github.com/elastic/gosigar" - "strings" -) - -func FindClickHouseProcesses() ([]gosigar.ProcArgs, error) { - pids := gosigar.ProcList{} - err := pids.Get() - if err != nil { - return nil, err - } - var clickhousePs []gosigar.ProcArgs - for _, pid := range pids.List { - args := gosigar.ProcArgs{} - if err := args.Get(pid); err != nil { - continue - } - if len(args.List) > 0 { - if strings.Contains(args.List[0], "clickhouse-server") { - clickhousePs = append(clickhousePs, args) - } - } - } - return clickhousePs, nil -} - -func FindConfigsFromClickHouseProcesses() ([]string, error) { - clickhouseProcesses, err := FindClickHouseProcesses() - if err != nil { - return nil, err - } - var configs []string - if len(clickhouseProcesses) > 0 { - // we have candidate matches - for _, ps := range clickhouseProcesses { - for _, arg := range ps.List { - if strings.Contains(arg, "--config") { - configFile := strings.ReplaceAll(arg, "--config-file=", "") - // containers receive config with --config - configFile = strings.ReplaceAll(configFile, "--config=", "") - configs = append(configs, configFile) - } - } - } - } - return configs, err -} diff --git a/programs/diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go deleted file mode 100644 index 9baaa559752..00000000000 --- a/programs/diagnostics/internal/platform/utils/process_test.go +++ /dev/null @@ -1,97 +0,0 @@ -//go:build !no_docker - -package utils_test - -import ( - "context" - "fmt" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -func getProcessesInContainer(t *testing.T, container testcontainers.Container) ([]string, error) { - result, reader, err := container.Exec(context.Background(), []string{"ps", "-aux"}) - if err != nil { - return nil, err - } - require.Zero(t, result) - require.NotNil(t, reader) - - b, err := io.ReadAll(reader) - if err != nil { - return nil, err - } - require.NotNil(t, b) - - lines := strings.Split(string(b), "\n") - - // discard PS header - return lines[1:], nil -} - -func TestFindClickHouseProcessesAndConfigs(t *testing.T) { - - t.Run("can find ClickHouse processes and configs", func(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - if err != nil { - fmt.Println("unable to read current directory", err) - os.Exit(1) - } - - // run a ClickHouse container that guarantees that it runs only for the duration of the test - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../../../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - - defer clickhouseContainer.Terminate(ctx) //nolint - - lines, err := getProcessesInContainer(t, clickhouseContainer) - require.Nil(t, err) - require.NotEmpty(t, lines) - - for _, line := range lines { - parts := strings.Fields(line) - if len(parts) < 11 { - continue - } - if !strings.Contains(parts[10], "clickhouse-server") { - continue - } - - require.Equal(t, "/usr/bin/clickhouse-server", parts[10]) - require.Equal(t, "--config-file=/etc/clickhouse-server/config.xml", parts[11]) - } - }) -} diff --git a/programs/diagnostics/internal/platform/utils/slices.go b/programs/diagnostics/internal/platform/utils/slices.go deleted file mode 100644 index cf5a5f97ce8..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices.go +++ /dev/null @@ -1,68 +0,0 @@ -package utils - -// Intersection of elements in s1 and s2 -func Intersection(s1, s2 []string) (inter []string) { - hash := make(map[string]bool) - for _, e := range s1 { - hash[e] = false - } - for _, e := range s2 { - // If elements present in the hashmap then append intersection list. - if val, ok := hash[e]; ok { - if !val { - // only add once - inter = append(inter, e) - hash[e] = true - } - } - } - return inter -} - -// Distinct returns elements in s1, not in s2 -func Distinct(s1, s2 []string) (distinct []string) { - hash := make(map[string]bool) - for _, e := range s2 { - hash[e] = true - } - for _, e := range s1 { - if _, ok := hash[e]; !ok { - distinct = append(distinct, e) - } - } - return distinct -} - -// Unique func Unique(s1 []string) (unique []string) returns unique elements in s1 -func Unique(s1 []string) (unique []string) { - hash := make(map[string]bool) - for _, e := range s1 { - if _, ok := hash[e]; !ok { - unique = append(unique, e) - } - hash[e] = true - } - return unique -} - -func Contains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - -func IndexOf(s []string, e string) int { - for i, a := range s { - if a == e { - return i - } - } - return -1 -} - -func Remove(slice []interface{}, s int) []interface{} { - return append(slice[:s], slice[s+1:]...) -} diff --git a/programs/diagnostics/internal/platform/utils/slices_test.go b/programs/diagnostics/internal/platform/utils/slices_test.go deleted file mode 100644 index ea5c1c81dcc..00000000000 --- a/programs/diagnostics/internal/platform/utils/slices_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package utils_test - -import ( - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" -) - -func TestIntersection(t *testing.T) { - t.Run("can perform intersection", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Intersection(setA, setB) - require.Len(t, setC, 2) - require.ElementsMatch(t, []string{"A", "C"}, setC) - }) -} - -func TestDistinct(t *testing.T) { - t.Run("can perform distinct", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - setB := []string{"A", "B", "F", "C", "G"} - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 3) - require.ElementsMatch(t, []string{"b", "D", "E"}, setC) - }) - - t.Run("can perform distinct on empty", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - var setB []string - setC := utils.Distinct(setA, setB) - require.Len(t, setC, 5) - require.ElementsMatch(t, []string{"A", "b", "C", "D", "E"}, setC) - }) -} - -func TestContains(t *testing.T) { - t.Run("can perform contains", func(t *testing.T) { - setA := []string{"A", "b", "C", "D", "E"} - require.True(t, utils.Contains(setA, "A")) - require.True(t, utils.Contains(setA, "b")) - require.True(t, utils.Contains(setA, "C")) - require.True(t, utils.Contains(setA, "D")) - require.True(t, utils.Contains(setA, "E")) - require.False(t, utils.Contains(setA, "B")) - }) -} - -func TestUnique(t *testing.T) { - - t.Run("can perform unique", func(t *testing.T) { - setA := []string{"A", "b", "D", "D", "E", "E", "A"} - setC := utils.Unique(setA) - require.Len(t, setC, 4) - require.ElementsMatch(t, []string{"A", "b", "D", "E"}, setC) - }) - - t.Run("can perform unique on empty", func(t *testing.T) { - var setA []string - setC := utils.Unique(setA) - require.Len(t, setC, 0) - }) -} diff --git a/programs/diagnostics/internal/platform/utils/time.go b/programs/diagnostics/internal/platform/utils/time.go deleted file mode 100644 index 622e92b873a..00000000000 --- a/programs/diagnostics/internal/platform/utils/time.go +++ /dev/null @@ -1,7 +0,0 @@ -package utils - -import "time" - -func MakeTimestamp() int64 { - return time.Now().UnixNano() / int64(time.Millisecond) -} diff --git a/programs/diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go deleted file mode 100644 index 9386a1d178b..00000000000 --- a/programs/diagnostics/internal/runner.go +++ /dev/null @@ -1,115 +0,0 @@ -package internal - -import ( - c "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - o "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" -) - -type runConfiguration struct { - id string - host string - port uint16 - username string - password string - output string - collectors []string - collectorConfigs map[string]config.Configuration - outputConfig config.Configuration -} - -func NewRunConfiguration(id string, host string, port uint16, username string, password string, output string, outputConfig config.Configuration, - collectors []string, collectorConfigs map[string]config.Configuration) *runConfiguration { - config := runConfiguration{ - id: id, - host: host, - port: port, - username: username, - password: password, - collectors: collectors, - output: output, - collectorConfigs: collectorConfigs, - outputConfig: outputConfig, - } - return &config -} - -func Capture(config *runConfiguration) { - bundles, err := collect(config) - if err != nil { - log.Fatal().Err(err).Msg("unable to perform collection") - } - log.Info().Msgf("collectors initialized") - if err = output(config, bundles); err != nil { - log.Fatal().Err(err).Msg("unable to create output") - } - log.Info().Msgf("bundle export complete") -} - -func collect(config *runConfiguration) (map[string]*data.DiagnosticBundle, error) { - resourceManager := platform.GetResourceManager() - err := resourceManager.Connect(config.host, config.port, config.username, config.password) - if err != nil { - // if we can't connect this is fatal - log.Fatal().Err(err).Msg("Unable to connect to database") - } - //grab the required connectors - we pass what we can - bundles := make(map[string]*data.DiagnosticBundle) - log.Info().Msgf("connection established") - //these store our collection errors and will be output in the bundle - var collectorErrors [][]interface{} - for _, collectorName := range config.collectors { - collectorConfig := config.collectorConfigs[collectorName] - log.Info().Msgf("initializing %s collector", collectorName) - collector, err := c.GetCollectorByName(collectorName) - if err != nil { - log.Error().Err(err).Msgf("Unable to fetch collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - continue - } - bundle, err := collector.Collect(collectorConfig) - if err != nil { - log.Error().Err(err).Msgf("Error in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - // this indicates a fatal error in the collector - continue - } - for _, fError := range bundle.Errors.Errors { - err = errors.Wrapf(fError, "Failure to collect frame in collector %s", collectorName) - collectorErrors = append(collectorErrors, []interface{}{err.Error()}) - log.Warn().Msg(err.Error()) - } - bundles[collectorName] = bundle - } - bundles["diag_trace"] = buildTraceBundle(collectorErrors) - return bundles, nil -} - -func output(config *runConfiguration, bundles map[string]*data.DiagnosticBundle) error { - log.Info().Msgf("attempting to export bundle using %s output...", config.output) - output, err := o.GetOutputByName(config.output) - if err != nil { - return err - } - frameErrors, err := output.Write(config.id, bundles, config.outputConfig) - // we report over failing hard on frame errors - up to the output to determine what is fatal via error - for _, fError := range frameErrors.Errors { - log.Warn().Msgf("failure to write frame in output %s - %s", config.output, fError) - } - return err -} - -func buildTraceBundle(collectorErrors [][]interface{}) *data.DiagnosticBundle { - errorBundle := data.DiagnosticBundle{ - Frames: map[string]data.Frame{ - "errors": data.NewMemoryFrame("errors", []string{"errors"}, collectorErrors), - }, - Errors: data.FrameErrors{}, - } - // add any other metrics from collection - return &errorBundle -} diff --git a/programs/diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go deleted file mode 100644 index 2369f8b3007..00000000000 --- a/programs/diagnostics/internal/runner_test.go +++ /dev/null @@ -1,130 +0,0 @@ -//go:build !no_docker - -package internal_test - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path" - "testing" - - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -// Execute a full default capture, with simple output, and check if a bundle is produced and it's not empty -func TestCapture(t *testing.T) { - // create a ClickHouse container - ctx := context.Background() - cwd, err := os.Getwd() - - if err != nil { - // can't test without container - panic(err) - } - // for now, we test against a hardcoded database-server version but we should make this a property - req := testcontainers.ContainerRequest{ - Image: fmt.Sprintf("clickhouse/clickhouse-server:%s", test.GetClickHouseTestVersion()), - ExposedPorts: []string{"9000/tcp"}, - WaitingFor: wait.ForLog("Ready for connections"), - Mounts: testcontainers.ContainerMounts{ - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/custom.xml"), - }, - Target: "/etc/clickhouse-server/config.d/custom.xml", - }, - { - Source: testcontainers.GenericBindMountSource{ - HostPath: path.Join(cwd, "../testdata/docker/admin.xml"), - }, - Target: "/etc/clickhouse-server/users.d/admin.xml", - }, - }, - } - clickhouseContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - // can't test without container - panic(err) - } - - p, _ := clickhouseContainer.MappedPort(ctx, "9000") - - t.Setenv("CLICKHOUSE_DB_PORT", p.Port()) - defer clickhouseContainer.Terminate(ctx) //nolint - - tmrDir := t.TempDir() - port := p.Int() - - // test a simple output exists - _, err = outputs.GetOutputByName("simple") - require.Nil(t, err) - // this relies on the simple out not changing its params - test will likely fail if so - outputConfig := config.Configuration{ - Params: []config.ConfigParam{ - config.StringParam{ - Value: tmrDir, - Param: config.NewParam("directory", "Directory in which to create dump. Defaults to the current directory.", false), - }, - config.StringOptions{ - Value: "csv", - Options: []string{"csv"}, - Param: config.NewParam("format", "Format of exported files", false), - }, - config.BoolParam{ - Value: true, - Param: config.NewParam("skip_archive", "Don't compress output to an archive", false), - }, - }, - } - // test default collectors - collectorNames := collectors.GetCollectorNames(true) - // grab all configs - only default will be used because of collectorNames - collectorConfigs, err := collectors.BuildConfigurationOptions() - require.Nil(t, err) - conf := internal.NewRunConfiguration("random", "localhost", uint16(port), "", "", "simple", outputConfig, collectorNames, collectorConfigs) - internal.Capture(conf) - outputDir := path.Join(tmrDir, "random") - _, err = os.Stat(outputDir) - require.Nil(t, err) - require.True(t, !os.IsNotExist(err)) - files, err := ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, 1) - outputDir = path.Join(outputDir, files[0].Name()) - // check we have a folder per collector i.e. collectorNames + diag_trace - files, err = ioutil.ReadDir(outputDir) - require.Nil(t, err) - require.Len(t, files, len(collectorNames)+1) - expectedFolders := append(collectorNames, "diag_trace") - for _, file := range files { - require.True(t, file.IsDir()) - utils.Contains(expectedFolders, file.Name()) - } - // we don't test the specific collector outputs but make sure something was written to system - systemFolder := path.Join(outputDir, "system") - files, err = ioutil.ReadDir(systemFolder) - require.Nil(t, err) - require.Greater(t, len(files), 0) - // test diag_trace - diagFolder := path.Join(outputDir, "diag_trace") - files, err = ioutil.ReadDir(diagFolder) - require.Nil(t, err) - require.Equal(t, 1, len(files)) - require.FileExists(t, path.Join(diagFolder, "errors.csv")) -} diff --git a/programs/diagnostics/testdata/configs/include/xml/server-include.xml b/programs/diagnostics/testdata/configs/include/xml/server-include.xml deleted file mode 100644 index 30e6587c935..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/server-include.xml +++ /dev/null @@ -1,8 +0,0 @@ - - 5000000 - - - - - 9008 - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/include/xml/user-include.xml b/programs/diagnostics/testdata/configs/include/xml/user-include.xml deleted file mode 100644 index b12b34a56bb..00000000000 --- a/programs/diagnostics/testdata/configs/include/xml/user-include.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - ::/0 - - default - default - REPLACE_ME - 1 - - - - ::/0 - - default - default - REPLACE_ME - 1 - - diff --git a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml deleted file mode 100644 index 903d7b6f733..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml +++ /dev/null @@ -1 +0,0 @@ -network_max: 5000000 diff --git a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml deleted file mode 100644 index 23b592507fa..00000000000 --- a/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml +++ /dev/null @@ -1,7 +0,0 @@ -test_user: - password: 'REPLACE_ME' - networks: - ip: '::/0' - profile: default - quota: default - access_management: 1 diff --git a/programs/diagnostics/testdata/configs/xml/config.xml b/programs/diagnostics/testdata/configs/xml/config.xml deleted file mode 100644 index eb7c70cf498..00000000000 --- a/programs/diagnostics/testdata/configs/xml/config.xml +++ /dev/null @@ -1,1195 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - REPLACE_ME - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - - - - s3 - https://storage.yandexcloud.net/my-bucket/root-path/ - REPLACE_ME - REPLACE_ME - -

Authorization: Bearer SOME-TOKEN
- your_base64_encoded_customer_key - - REPLACE_ME - REPLACE_ME - true - - http://proxy1 - http://proxy2 - - 10000 - 5000 - 10 - 4 - 1000 - /var/lib/clickhouse/disks/s3/ - false - - - - diff --git a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml b/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml deleted file mode 100644 index 242a6a4b02e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - REPLACE_ME - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/configs/xml/users.xml b/programs/diagnostics/testdata/configs/xml/users.xml deleted file mode 100644 index cd5f17e922e..00000000000 --- a/programs/diagnostics/testdata/configs/xml/users.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - ../include/xml/user-include.xml - - - - - - 10000000000 - - random - 1 - - - - 1 - - - - - - - - - - REPLACE_ME - - ::/0 - - - default - - default - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/programs/diagnostics/testdata/configs/yaml/config.yaml b/programs/diagnostics/testdata/configs/yaml/config.yaml deleted file mode 100644 index e577a99e675..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/config.yaml +++ /dev/null @@ -1,927 +0,0 @@ -# This is an example of a configuration file "config.xml" rewritten in YAML -# You can read this documentation for detailed information about YAML configuration: -# https://clickhouse.com/docs/en/operations/configuration-files/ - -# NOTE: User and query level settings are set up in "users.yaml" file. -# If you have accidentally specified user-level settings here, server won't start. -# You can either move the settings to the right place inside "users.xml" file -# or add skip_check_for_incorrect_settings: 1 here. -include_from: "../include/yaml/server-include.yaml" -logger: - # Possible levels [1]: - # - none (turns off logging) - # - fatal - # - critical - # - error - # - warning - # - notice - # - information - # - debug - # - trace - # [1]: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105-L114 - level: trace - log: /var/log/clickhouse-server/clickhouse-server.log - errorlog: /var/log/clickhouse-server/clickhouse-server.err.log - # Rotation policy - # See https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/FileChannel.h#L54-L85 - size: 1000M - count: 10 - # console: 1 - # Default behavior is autodetection (log to console if not daemon mode and is tty) - - # Per level overrides (legacy): - # For example to suppress logging of the ConfigReloader you can use: - # NOTE: levels.logger is reserved, see below. - # levels: - # ConfigReloader: none - - # Per level overrides: - # For example to suppress logging of the RBAC for default user you can use: - # (But please note that the logger name maybe changed from version to version, even after minor upgrade) - # levels: - # - logger: - # name: 'ContextAccess (default)' - # level: none - # - logger: - # name: 'DatabaseOrdinary (test)' - # level: none - -# It is the name that will be shown in the clickhouse-client. -# By default, anything with "production" will be highlighted in red in query prompt. -# display_name: production - -# Port for HTTP API. See also 'https_port' for secure connections. -# This interface is also used by ODBC and JDBC drivers (DataGrip, Dbeaver, ...) -# and by most of web interfaces (embedded UI, Grafana, Redash, ...). -http_port: 8123 - -# Port for interaction by native protocol with: -# - clickhouse-client and other native ClickHouse tools (clickhouse-benchmark); -# - clickhouse-server with other clickhouse-servers for distributed query processing; -# - ClickHouse drivers and applications supporting native protocol -# (this protocol is also informally called as "the TCP protocol"); -# See also 'tcp_port_secure' for secure connections. -tcp_port: 9000 - -# Compatibility with MySQL protocol. -# ClickHouse will pretend to be MySQL for applications connecting to this port. -mysql_port: 9004 - -# Compatibility with PostgreSQL protocol. -# ClickHouse will pretend to be PostgreSQL for applications connecting to this port. -postgresql_port: 9005 - -# HTTP API with TLS (HTTPS). -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# https_port: 8443 - -# Native interface with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# tcp_port_secure: 9440 - -# Native interface wrapped with PROXYv1 protocol -# PROXYv1 header sent for every connection. -# ClickHouse will extract information about proxy-forwarded client address from the header. -# tcp_with_proxy_port: 9011 - -# Port for communication between replicas. Used for data exchange. -# It provides low-level data access between servers. -# This port should not be accessible from untrusted networks. -# See also 'interserver_http_credentials'. -# Data transferred over connections to this port should not go through untrusted networks. -# See also 'interserver_https_port'. -interserver_http_port: 9009 - -# Port for communication between replicas with TLS. -# You have to configure certificate to enable this interface. -# See the openSSL section below. -# See also 'interserver_http_credentials'. -# interserver_https_port: 9010 - -# Hostname that is used by other replicas to request this server. -# If not specified, than it is determined analogous to 'hostname -f' command. -# This setting could be used to switch replication to another network interface -# (the server may be connected to multiple networks via multiple addresses) -# interserver_http_host: example.yandex.ru - -# You can specify credentials for authenthication between replicas. -# This is required when interserver_https_port is accessible from untrusted networks, -# and also recommended to avoid SSRF attacks from possibly compromised services in your network. -# interserver_http_credentials: -# user: interserver -# password: '' - -# Listen specified address. -# Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. -# Notes: -# If you open connections from wildcard address, make sure that at least one of the following measures applied: -# - server is protected by firewall and not accessible from untrusted networks; -# - all users are restricted to subset of network addresses (see users.xml); -# - all users have strong passwords, only secure (TLS) interfaces are accessible, or connections are only made via TLS interfaces. -# - users without password have readonly access. -# See also: https://www.shodan.io/search?query=clickhouse -# listen_host: '::' - -# Same for hosts without support for IPv6: -# listen_host: 0.0.0.0 - -# Default values - try listen localhost on IPv4 and IPv6. -# listen_host: '::1' -# listen_host: 127.0.0.1 - -# Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -# listen_try: 0 - -# Allow multiple servers to listen on the same address:port. This is not recommended. -# listen_reuse_port: 0 - -# listen_backlog: 64 -max_connections: 4096 - -# For 'Connection: keep-alive' in HTTP 1.1 -keep_alive_timeout: 3 - -# gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) -# grpc_port: 9100 -grpc: - enable_ssl: false - - # The following two files are used only if enable_ssl=1 - ssl_cert_file: /path/to/ssl_cert_file - ssl_key_file: /path/to/ssl_key_file - - # Whether server will request client for a certificate - ssl_require_client_auth: false - - # The following file is used only if ssl_require_client_auth=1 - ssl_ca_cert_file: /path/to/ssl_ca_cert_file - - # Default compression algorithm (applied if client doesn't specify another algorithm). - # Supported algorithms: none, deflate, gzip, stream_gzip - compression: deflate - - # Default compression level (applied if client doesn't specify another level). - # Supported levels: none, low, medium, high - compression_level: medium - - # Send/receive message size limits in bytes. -1 means unlimited - max_send_message_size: -1 - max_receive_message_size: -1 - - # Enable if you want very detailed logs - verbose_logs: false - -# Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -openSSL: - server: - # Used for https server AND secure tcp port - # openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt - certificateFile: /etc/clickhouse-server/server.crt - privateKeyFile: /etc/clickhouse-server/server.key - - # dhparams are optional. You can delete the dhParamsFile: element. - # To generate dhparams, use the following command: - # openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 - # Only file format with BEGIN DH PARAMETERS is supported. - dhParamsFile: /etc/clickhouse-server/dhparam.pem - verificationMode: none - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - client: - # Used for connecting to https dictionary source and secured Zookeeper communication - loadDefaultCAFile: true - cacheSessions: true - disableProtocols: 'sslv2,sslv3' - preferServerCiphers: true - - # Use for self-signed: verificationMode: none - invalidCertificateHandler: - # Use for self-signed: name: AcceptCertificateHandler - name: RejectCertificateHandler - -# Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 -# http_server_default_response: |- -#
- -# Maximum number of concurrent queries. -max_concurrent_queries: 100 - -# Maximum memory usage (resident set size) for server process. -# Zero value or unset means default. Default is "max_server_memory_usage_to_ram_ratio" of available physical RAM. -# If the value is larger than "max_server_memory_usage_to_ram_ratio" of available physical RAM, it will be cut down. - -# The constraint is checked on query execution time. -# If a query tries to allocate memory and the current memory usage plus allocation is greater -# than specified threshold, exception will be thrown. - -# It is not practical to set this constraint to small values like just a few gigabytes, -# because memory allocator will keep this amount of memory in caches and the server will deny service of queries. -max_server_memory_usage: 0 - -# Maximum number of threads in the Global thread pool. -# This will default to a maximum of 10000 threads if not specified. -# This setting will be useful in scenarios where there are a large number -# of distributed queries that are running concurrently but are idling most -# of the time, in which case a higher number of threads might be required. -max_thread_pool_size: 10000 - -# On memory constrained environments you may have to set this to value larger than 1. -max_server_memory_usage_to_ram_ratio: 0.9 - -# Simple server-wide memory profiler. Collect a stack trace at every peak allocation step (in bytes). -# Data will be stored in system.trace_log table with query_id = empty string. -# Zero means disabled. -total_memory_profiler_step: 4194304 - -# Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. -# The probability is for every alloc/free regardless to the size of the allocation. -# Note that sampling happens only when the amount of untracked memory exceeds the untracked memory limit, -# which is 4 MiB by default but can be lowered if 'total_memory_profiler_step' is lowered. -# You may want to set 'total_memory_profiler_step' to 1 for extra fine grained sampling. -total_memory_tracker_sample_probability: 0 - -# Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve -# correct maximum value. -# max_open_files: 262144 - -# Size of cache of uncompressed blocks of data, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# Cache is used when 'use_uncompressed_cache' user setting turned on (off by default). -# Uncompressed cache is advantageous only for very short queries and in rare cases. - -# Note: uncompressed cache can be pointless for lz4, because memory bandwidth -# is slower than multi-core decompression on some server configurations. -# Enabling it can sometimes paradoxically make queries slower. -uncompressed_cache_size: 8589934592 - -# Approximate size of mark cache, used in tables of MergeTree family. -# In bytes. Cache is single for server. Memory is allocated only on demand. -# You should not lower this value. -mark_cache_size: 5368709120 - -# If you enable the `min_bytes_to_use_mmap_io` setting, -# the data in MergeTree tables can be read with mmap to avoid copying from kernel to userspace. -# It makes sense only for large files and helps only if data reside in page cache. -# To avoid frequent open/mmap/munmap/close calls (which are very expensive due to consequent page faults) -# and to reuse mappings from several threads and queries, -# the cache of mapped files is maintained. Its size is the number of mapped regions (usually equal to the number of mapped files). -# The amount of data in mapped files can be monitored -# in system.metrics, system.metric_log by the MMappedFiles, MMappedFileBytes metrics -# and in system.asynchronous_metrics, system.asynchronous_metrics_log by the MMapCacheCells metric, -# and also in system.events, system.processes, system.query_log, system.query_thread_log, system.query_views_log by the -# CreatedReadBufferMMap, CreatedReadBufferMMapFailed, MMappedFileCacheHits, MMappedFileCacheMisses events. -# Note that the amount of data in mapped files does not consume memory directly and is not accounted -# in query or server memory usage - because this memory can be discarded similar to OS page cache. -# The cache is dropped (the files are closed) automatically on removal of old parts in MergeTree, -# also it can be dropped manually by the SYSTEM DROP MMAP CACHE query. -mmap_cache_size: 1000 - -# Cache size in bytes for compiled expressions. -compiled_expression_cache_size: 134217728 - -# Cache size in elements for compiled expressions. -compiled_expression_cache_elements_size: 10000 - -# Path to data directory, with trailing slash. -path: /var/lib/clickhouse/ - -# Path to temporary data for processing hard queries. -tmp_path: /var/lib/clickhouse/tmp/ - -# Policy from the for the temporary files. -# If not set is used, otherwise is ignored. - -# Notes: -# - move_factor is ignored -# - keep_free_space_bytes is ignored -# - max_data_part_size_bytes is ignored -# - you must have exactly one volume in that policy -# tmp_policy: tmp - -# Directory with user provided files that are accessible by 'file' table function. -user_files_path: /var/lib/clickhouse/user_files/ - -# LDAP server definitions. -ldap_servers: '' - -# List LDAP servers with their connection parameters here to later 1) use them as authenticators for dedicated local users, -# who have 'ldap' authentication mechanism specified instead of 'password', or to 2) use them as remote user directories. -# Parameters: -# host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty. -# port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise. -# bind_dn - template used to construct the DN to bind to. -# The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual -# user name during each authentication attempt. -# user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user. -# This is mainly used in search filters for further role mapping when the server is Active Directory. The -# resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default, -# user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected -# user DN value. -# base_dn - template used to construct the base DN for the LDAP search. -# The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings -# of the template with the actual user name and bind DN during the LDAP search. -# scope - scope of the LDAP search. -# Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# search_filter - template used to construct the search filter for the LDAP search. -# The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' -# substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. -# Note, that the special characters must be escaped properly in XML. -# verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed -# to be successfully authenticated for all consecutive requests without contacting the LDAP server. -# Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. -# enable_tls - flag to trigger use of secure connection to the LDAP server. -# Specify 'no' for plain text (ldap://) protocol (not recommended). -# Specify 'yes' for LDAP over SSL/TLS (ldaps://) protocol (recommended, the default). -# Specify 'starttls' for legacy StartTLS protocol (plain text (ldap://) protocol, upgraded to TLS). -# tls_minimum_protocol_version - the minimum protocol version of SSL/TLS. -# Accepted values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2' (the default). -# tls_require_cert - SSL/TLS peer certificate verification behavior. -# Accepted values are: 'never', 'allow', 'try', 'demand' (the default). -# tls_cert_file - path to certificate file. -# tls_key_file - path to certificate key file. -# tls_ca_cert_file - path to CA certificate file. -# tls_ca_cert_dir - path to the directory containing CA certificates. -# tls_cipher_suite - allowed cipher suite (in OpenSSL notation). -# Example: -# my_ldap_server: -# host: localhost -# port: 636 -# bind_dn: 'uid={user_name},ou=users,dc=example,dc=com' -# verification_cooldown: 300 -# enable_tls: yes -# tls_minimum_protocol_version: tls1.2 -# tls_require_cert: demand -# tls_cert_file: /path/to/tls_cert_file -# tls_key_file: /path/to/tls_key_file -# tls_ca_cert_file: /path/to/tls_ca_cert_file -# tls_ca_cert_dir: /path/to/tls_ca_cert_dir -# tls_cipher_suite: ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 - -# Example (typical Active Directory with configured user DN detection for further role mapping): -# my_ad_server: -# host: localhost -# port: 389 -# bind_dn: 'EXAMPLE\{user_name}' -# user_dn_detection: -# base_dn: CN=Users,DC=example,DC=com -# search_filter: '(&(objectClass=user)(sAMAccountName={user_name}))' -# enable_tls: no - -# To enable Kerberos authentication support for HTTP requests (GSS-SPNEGO), for those users who are explicitly configured -# to authenticate via Kerberos, define a single 'kerberos' section here. -# Parameters: -# principal - canonical service principal name, that will be acquired and used when accepting security contexts. -# This parameter is optional, if omitted, the default principal will be used. -# This parameter cannot be specified together with 'realm' parameter. -# realm - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it. -# This parameter is optional, if omitted, no additional filtering by realm will be applied. -# This parameter cannot be specified together with 'principal' parameter. -# Example: -# kerberos: '' - -# Example: -# kerberos: -# principal: HTTP/clickhouse.example.com@EXAMPLE.COM - -# Example: -# kerberos: -# realm: EXAMPLE.COM - -# Sources to read users, roles, access rights, profiles of settings, quotas. -user_directories: - users_xml: - # Path to configuration file with predefined users. - path: users.yaml - local_directory: - # Path to folder where users created by SQL commands are stored. - path: /var/lib/clickhouse/access/ - -# # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section -# # with the following parameters: -# # server - one of LDAP server names defined in 'ldap_servers' config section above. -# # This parameter is mandatory and cannot be empty. -# # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. -# # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any -# # actions after authentication. -# # role_mapping - section with LDAP search parameters and mapping rules. -# # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the -# # name of the logged in user. For each entry found during that search, the value of the specified attribute is -# # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the -# # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by -# # CREATE ROLE command. -# # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be -# # applied. -# # base_dn - template used to construct the base DN for the LDAP search. -# # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' -# # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. -# # scope - scope of the LDAP search. -# # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). -# # search_filter - template used to construct the search filter for the LDAP search. -# # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and -# # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during -# # each LDAP search. -# # Note, that the special characters must be escaped properly in XML. -# # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. -# # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by -# # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated -# # as local role names. Empty, by default. -# # Example: -# # ldap: -# # server: my_ldap_server -# # roles: -# # my_local_role1: '' -# # my_local_role2: '' -# # role_mapping: -# # base_dn: 'ou=groups,dc=example,dc=com' -# # scope: subtree -# # search_filter: '(&(objectClass=groupOfNames)(member={bind_dn}))' -# # attribute: cn -# # prefix: clickhouse_ -# # Example (typical Active Directory with role mapping that relies on the detected user DN): -# # ldap: -# # server: my_ad_server -# # role_mapping: -# # base_dn: 'CN=Users,DC=example,DC=com' -# # attribute: CN -# # scope: subtree -# # search_filter: '(&(objectClass=group)(member={user_dn}))' -# # prefix: clickhouse_ - -# Default profile of settings. -default_profile: default - -# Comma-separated list of prefixes for user-defined settings. -# custom_settings_prefixes: '' -# System profile of settings. This settings are used by internal processes (Distributed DDL worker and so on). -# system_profile: default - -# Buffer profile of settings. -# This settings are used by Buffer storage to flush data to the underlying table. -# Default: used from system_profile directive. -# buffer_profile: default - -# Default database. -default_database: default - -# Server time zone could be set here. - -# Time zone is used when converting between String and DateTime types, -# when printing DateTime in text formats and parsing DateTime from text, -# it is used in date and time related functions, if specific time zone was not passed as an argument. - -# Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan. -# If not specified, system time zone at server startup is used. - -# Please note, that server could display time zone alias instead of specified name. -# Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC. -# timezone: Europe/Moscow - -# You can specify umask here (see "man umask"). Server will apply it on startup. -# Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read). -# umask: 022 - -# Perform mlockall after startup to lower first queries latency -# and to prevent clickhouse executable from being paged out under high IO load. -# Enabling this option is recommended but will lead to increased startup time for up to a few seconds. -mlock_executable: true - -# Reallocate memory for machine code ("text") using huge pages. Highly experimental. -remap_executable: false - -# Uncomment below in order to use JDBC table engine and function. -# To install and run JDBC bridge in background: -# * [Debian/Ubuntu] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb -# clickhouse-jdbc-bridge & -# * [CentOS/RHEL] -# export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge -# export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') -# wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm -# clickhouse-jdbc-bridge & -# Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - -# jdbc_bridge: -# host: 127.0.0.1 -# port: 9019 - -# Configuration of clusters that could be used in Distributed tables. -# https://clickhouse.com/docs/en/operations/table_engines/distributed/ -remote_servers: - # Test only shard config for testing distributed storage - test_shard_localhost: - # Inter-server per-cluster secret for Distributed queries - # default: no secret (no authentication will be performed) - - # If set, then Distributed queries will be validated on shards, so at least: - # - such cluster should exist on the shard, - # - such cluster should have the same secret. - - # And also (and which is more important), the initial_user will - # be used as current user for the query. - - # Right now the protocol is pretty simple and it only takes into account: - # - cluster name - # - query - - # Also it will be nice if the following will be implemented: - # - source hostname (see interserver_http_host), but then it will depends from DNS, - # it can use IP address instead, but then the you need to get correct on the initiator node. - # - target hostname / ip address (same notes as for source hostname) - # - time-based security tokens - secret: 'REPLACE_ME' - shard: - # Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). - # internal_replication: false - # Optional. Shard weight when writing data. Default: 1. - # weight: 1 - replica: - host: localhost - port: 9000 - # Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). - # priority: 1 - -# The list of hosts allowed to use in URL-related storage engines and table functions. -# If this section is not present in configuration, all hosts are allowed. -# remote_url_allow_hosts: - -# Host should be specified exactly as in URL. The name is checked before DNS resolution. -# Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts. -# If port is explicitly specified in URL, the host:port is checked as a whole. -# If host specified here without port, any port with this host allowed. -# "yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed. -# If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]". -# If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked. - -# Regular expression can be specified. RE2 engine is used for regexps. -# Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter -# (forgetting to do so is a common source of error). - -# If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file. -# By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element. -# Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. - -# ZooKeeper is used to store metadata about replicas, when using Replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ - -# zookeeper: -# - node: -# host: example1 -# port: 2181 -# - node: -# host: example2 -# port: 2181 -# - node: -# host: example3 -# port: 2181 - -# Substitutions for parameters of replicated tables. -# Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables -# macros: -# shard: 01 -# replica: example01-01-1 - -# Reloading interval for embedded dictionaries, in seconds. Default: 3600. -builtin_dictionaries_reload_interval: 3600 - -# Maximum session timeout, in seconds. Default: 3600. -max_session_timeout: 3600 - -# Default session timeout, in seconds. Default: 60. -default_session_timeout: 60 - -# Sending data to Graphite for monitoring. Several sections can be defined. -# interval - send every X second -# root_path - prefix for keys -# hostname_in_path - append hostname to root_path (default = true) -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 60 -# root_path: one_min -# hostname_in_path: true - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: true - -# graphite: -# host: localhost -# port: 42000 -# timeout: 0.1 -# interval: 1 -# root_path: one_sec - -# metrics: true -# events: true -# events_cumulative: false -# asynchronous_metrics: false - -# Serve endpoint for Prometheus monitoring. -# endpoint - mertics path (relative to root, statring with "/") -# port - port to setup server. If not defined or 0 than http_port used -# metrics - send data from table system.metrics -# events - send data from table system.events -# asynchronous_metrics - send data from table system.asynchronous_metrics - -# prometheus: -# endpoint: /metrics -# port: 9363 - -# metrics: true -# events: true -# asynchronous_metrics: true - -# Query log. Used only for queries with setting log_queries = 1. -query_log: - # What table to insert data. If table is not exist, it will be created. - # When query log structure is changed after system update, - # then old table will be renamed and new table will be created automatically. - database: system - table: query_log - - # PARTITION BY expr: https://clickhouse.com/docs/en/table_engines/mergetree-family/custom_partitioning_key/ - # Example: - # event_date - # toMonday(event_date) - # toYYYYMM(event_date) - # toStartOfHour(event_time) - partition_by: toYYYYMM(event_date) - - # Table TTL specification: https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl - # Example: - # event_date + INTERVAL 1 WEEK - # event_date + INTERVAL 7 DAY DELETE - # event_date + INTERVAL 2 WEEK TO DISK 'bbb' - - # ttl: 'event_date + INTERVAL 30 DAY DELETE' - - # Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters, - # Example: engine: 'ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024' - - # Interval of flushing data. - flush_interval_milliseconds: 7500 - -# Trace log. Stores stack traces collected by query profilers. -# See query_profiler_real_time_period_ns and query_profiler_cpu_time_period_ns settings. -trace_log: - database: system - table: trace_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query thread log. Has information about all threads participated in query execution. -# Used only for queries with setting log_query_threads = 1. -query_thread_log: - database: system - table: query_thread_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Query views log. Has information about all dependent views associated with a query. -# Used only for queries with setting log_query_views = 1. -query_views_log: - database: system - table: query_views_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment if use part log. -# Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads). -part_log: - database: system - table: part_log - partition_by: toYYYYMM(event_date) - flush_interval_milliseconds: 7500 - -# Uncomment to write text log into table. -# Text log contains all information from usual server log but stores it in structured and efficient way. -# The level of the messages that goes to the table can be limited (), if not specified all messages will go to the table. -# text_log: -# database: system -# table: text_log -# flush_interval_milliseconds: 7500 -# level: '' - -# Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -metric_log: - database: system - table: metric_log - flush_interval_milliseconds: 7500 - collect_interval_milliseconds: 1000 - -# Asynchronous metric log contains values of metrics from -# system.asynchronous_metrics. -asynchronous_metric_log: - database: system - table: asynchronous_metric_log - - # Asynchronous metrics are updated once a minute, so there is - # no need to flush more often. - flush_interval_milliseconds: 60000 - -# OpenTelemetry log contains OpenTelemetry trace spans. -opentelemetry_span_log: - - # The default table creation code is insufficient, this spec - # is a workaround. There is no 'event_time' for this log, but two times, - # start and finish. It is sorted by finish time, to avoid inserting - # data too far away in the past (probably we can sometimes insert a span - # that is seconds earlier than the last span in the table, due to a race - # between several spans inserted in parallel). This gives the spans a - # global order that we can use to e.g. retry insertion into some external - # system. - engine: |- - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - database: system - table: opentelemetry_span_log - flush_interval_milliseconds: 7500 - -# Crash log. Stores stack traces for fatal errors. -# This table is normally empty. -crash_log: - database: system - table: crash_log - partition_by: '' - flush_interval_milliseconds: 1000 - -# Parameters for embedded dictionaries, used in Yandex.Metrica. -# See https://clickhouse.com/docs/en/dicts/internal_dicts/ - -# Path to file with region hierarchy. -# path_to_regions_hierarchy_file: /opt/geo/regions_hierarchy.txt - -# Path to directory with files containing names of regions -# path_to_regions_names_files: /opt/geo/ - - -# top_level_domains_path: /var/lib/clickhouse/top_level_domains/ -# Custom TLD lists. -# Format: name: /path/to/file - -# Changes will not be applied w/o server restart. -# Path to the list is under top_level_domains_path (see above). -top_level_domains_lists: '' - -# public_suffix_list: /path/to/public_suffix_list.dat - -# Configuration of external dictionaries. See: -# https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts -dictionaries_config: '*_dictionary.xml' - -# Uncomment if you want data to be compressed 30-100% better. -# Don't do that if you just started using ClickHouse. - -# compression: -# # Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. -# case: -# Conditions. All must be satisfied. Some conditions may be omitted. -# # min_part_size: 10000000000 # Min part size in bytes. -# # min_part_size_ratio: 0.01 # Min size of part relative to whole table size. -# # What compression method to use. -# method: zstd - -# Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster. -# Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -distributed_ddl: - # Path in ZooKeeper to queue with DDL queries - path: /clickhouse/task_queue/ddl - - # Settings from this profile will be used to execute DDL queries - # profile: default - - # Controls how much ON CLUSTER queries can be run simultaneously. - # pool_size: 1 - - # Cleanup settings (active tasks will not be removed) - - # Controls task TTL (default 1 week) - # task_max_lifetime: 604800 - - # Controls how often cleanup should be performed (in seconds) - # cleanup_delay_period: 60 - - # Controls how many tasks could be in the queue - # max_tasks_in_queue: 1000 - -# Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -# merge_tree: -# max_suspicious_broken_parts: 5 - -# Protection from accidental DROP. -# If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query. -# If you want do delete one table and don't want to change clickhouse-server config, you could create special file /flags/force_drop_table and make DROP once. -# By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables. -# The same for max_partition_size_to_drop. -# Uncomment to disable protection. - -# max_table_size_to_drop: 0 -# max_partition_size_to_drop: 0 - -# Example of parameters for GraphiteMergeTree table engine -graphite_rollup_example: - pattern: - regexp: click_cost - function: any - retention: - - age: 0 - precision: 3600 - - age: 86400 - precision: 60 - default: - function: max - retention: - - age: 0 - precision: 60 - - age: 3600 - precision: 300 - - age: 86400 - precision: 3600 - -# Directory in containing schema files for various input formats. -# The directory will be created if it doesn't exist. -format_schema_path: /var/lib/clickhouse/format_schemas/ - -# Default query masking rules, matching lines would be replaced with something else in the logs -# (both text logs and system.query_log). -# name - name for the rule (optional) -# regexp - RE2 compatible regular expression (mandatory) -# replace - substitution string for sensitive data (optional, by default - six asterisks) -query_masking_rules: - rule: - name: hide encrypt/decrypt arguments - regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)' - # or more secure, but also more invasive: - # (aes_\w+)\s*\(.*\) - replace: \1(???) - -# Uncomment to use custom http handlers. -# rules are checked from top to bottom, first match runs the handler -# url - to match request URL, you can use 'regex:' prefix to use regex match(optional) -# methods - to match request method, you can use commas to separate multiple method matches(optional) -# headers - to match request headers, match each child element(child element name is header name), you can use 'regex:' prefix to use regex match(optional) -# handler is request handler -# type - supported types: static, dynamic_query_handler, predefined_query_handler -# query - use with predefined_query_handler type, executes query when the handler is called -# query_param_name - use with dynamic_query_handler type, extracts and executes the value corresponding to the value in HTTP request params -# status - use with static type, response status code -# content_type - use with static type, response content-type -# response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client. - -# http_handlers: -# - rule: -# url: / -# methods: POST,GET -# headers: -# pragma: no-cache -# handler: -# type: dynamic_query_handler -# query_param_name: query -# - rule: -# url: /predefined_query -# methods: POST,GET -# handler: -# type: predefined_query_handler -# query: 'SELECT * FROM system.settings' -# - rule: -# handler: -# type: static -# status: 200 -# content_type: 'text/plain; charset=UTF-8' -# response_content: config://http_server_default_response - -send_crash_reports: - # Changing to true allows sending crash reports to - # the ClickHouse core developers team via Sentry https://sentry.io - # Doing so at least in pre-production environments is highly appreciated - enabled: false - # Change to true if you don't feel comfortable attaching the server hostname to the crash report - anonymize: false - # Default endpoint should be changed to different Sentry DSN only if you have - # some in-house engineers or hired consultants who're going to debug ClickHouse issues for you - endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277' - # Uncomment to disable ClickHouse internal DNS caching. - # disable_internal_dns_cache: 1 - -storage_configuration: - disks: - s3: - secret_access_key: REPLACE_ME - access_key_id: 'REPLACE_ME' diff --git a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml b/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml deleted file mode 100644 index c27bb7cb071..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password_sha256_hex: "REPLACE_ME" diff --git a/programs/diagnostics/testdata/configs/yaml/users.yaml b/programs/diagnostics/testdata/configs/yaml/users.yaml deleted file mode 100644 index 82f2d67f2a4..00000000000 --- a/programs/diagnostics/testdata/configs/yaml/users.yaml +++ /dev/null @@ -1,47 +0,0 @@ -include_from: "../include/yaml/user-include.yaml" -# Profiles of settings. -profiles: - # Default settings. - default: - # Maximum memory usage for processing single query, in bytes. - max_memory_usage: 10000000000 - load_balancing: random - - # Profile that allows only read queries. - readonly: - readonly: 1 - -# Users and ACL. -users: - # If user name was not specified, 'default' user is used. - default: - - password: 'REPLACE_ME' - - networks: - ip: '::/0' - - # Settings profile for user. - profile: default - - # Quota for user. - quota: default - - # User can create other users and grant rights to them. - # access_management: 1 - -# Quotas. -quotas: - # Name of quota. - default: - # Limits for time interval. You could specify many intervals with different limits. - interval: - # Length of interval. - duration: 3600 - - # No limits. Just calculate resource usage for time interval. - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 diff --git a/programs/diagnostics/testdata/configs/yandex_xml/config.xml b/programs/diagnostics/testdata/configs/yandex_xml/config.xml deleted file mode 100644 index 40d1fa34b1a..00000000000 --- a/programs/diagnostics/testdata/configs/yandex_xml/config.xml +++ /dev/null @@ -1,1167 +0,0 @@ - - - ../include/xml/server-include.xml - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - - 1000M - 10 - - - - - - - - - - - - - - - - - 8123 - - - 9000 - - - 9004 - - - 9005 - - - - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4096 - - - 3 - - - - - false - - - /path/to/ssl_cert_file - /path/to/ssl_key_file - - - false - - - /path/to/ssl_ca_cert_file - - - deflate - - - medium - - - -1 - -1 - - - false - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 100 - - - 0 - - - - 10000 - - - 0.9 - - - 4194304 - - - 0 - - - - - - 8589934592 - - - 5368709120 - - - - 1000 - - - 134217728 - - - 10000 - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - - - - /var/lib/clickhouse/user_files/ - - - - - - - - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - - - default - - - - - - - - - - - - default - - - - - - - - - true - - - false - - ' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb - clickhouse-jdbc-bridge & - - * [CentOS/RHEL] - export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge - export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') - wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm - clickhouse-jdbc-bridge & - - Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. - ]]> - - - - - - - - - - - - - - - - localhost - 9000 - - - - - - - - - - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - - - - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - system - metric_log
- 7500 - 1000 - - - - - system - asynchronous_metric_log
- - 7000 - - - - - - - engine MergeTree - partition by toYYYYMM(finish_date) - order by (finish_date, finish_time_us, trace_id) - - system - opentelemetry_span_log
- 7500 - - - - - - system - crash_log
- - - 1000 - - - - - system - session_log
- - toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - - - - - *_dictionary.xml - - - *_function.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - - - - - - - - - - false - - false - - - https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - - - - - - - diff --git a/programs/diagnostics/testdata/docker/admin.xml b/programs/diagnostics/testdata/docker/admin.xml deleted file mode 100644 index 76aa670dcfe..00000000000 --- a/programs/diagnostics/testdata/docker/admin.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - 1 - - - - - 1 - - - \ No newline at end of file diff --git a/programs/diagnostics/testdata/docker/custom.xml b/programs/diagnostics/testdata/docker/custom.xml deleted file mode 100644 index bc1051178ca..00000000000 --- a/programs/diagnostics/testdata/docker/custom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - :: - 0.0.0.0 - 1 - - 1 - - diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log deleted file mode 100644 index 1a1768fe87e..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log +++ /dev/null @@ -1,10 +0,0 @@ -2021.12.13 10:12:26.940169 [ 38398 ] {} Access(local directory): File /var/lib/clickhouse/access/users.list doesn't exist -2021.12.13 10:12:26.940204 [ 38398 ] {} Access(local directory): Recovering lists in directory /var/lib/clickhouse/access/ -2021.12.13 10:12:40.649453 [ 38445 ] {} Access(user directories): from: 127.0.0.1, user: default: Authentication failed: Code: 193. DB::Exception: Invalid credentials. (WRONG_PASSWORD), Stack trace (when copying this message, always include the lines below): - -0. DB::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int, bool) @ 0x9b722d4 in /usr/bin/clickhouse -1. DB::IAccessStorage::throwInvalidCredentials() @ 0x119d9b27 in /usr/bin/clickhouse -2. DB::IAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119d98d7 in /usr/bin/clickhouse -3. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse -4. DB::MultipleAccessStorage::loginImpl(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&) const @ 0x119ff93c in /usr/bin/clickhouse -5. DB::IAccessStorage::login(DB::Credentials const&, Poco::Net::IPAddress const&, DB::ExternalAuthenticators const&, bool) const @ 0x119d9084 in /usr/bin/clickhouse diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log deleted file mode 100644 index f6abe7764ba..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log +++ /dev/null @@ -1,10 +0,0 @@ -2022.02.02 14:49:32.458680 [ 200404 ] {} DiskLocal: Reserving 2.47 MiB on disk `default`, having unreserved 1.56 TiB. -2022.02.02 14:49:32.459086 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Merging 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 into Wide -2022.02.02 14:49:32.459201 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::PrepareStage: Selected MergeAlgorithm: Horizontal -2022.02.02 14:49:32.459262 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 159 marks from part 202202_147058_147549_343, total 1289014 rows starting from the beginning of the part -2022.02.02 14:49:32.459614 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTreeSequentialSource: Reading 2 marks from part 202202_147550_147550_0, total 2618 rows starting from the beginning of the part -2022.02.02 14:49:32.507755 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} MergeTask::MergeProjectionsStage: Merge sorted 1291632 rows, containing 5 columns (5 merged, 0 gathered) in 0.048711404 sec., 26516008.448452853 rows/sec., 639.52 MiB/sec. -2022.02.02 14:49:32.508332 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c): Renaming temporary part tmp_merge_202202_147058_147550_344 to 202202_147058_147550_344. -2022.02.02 14:49:32.508406 [ 200359 ] {de87df8b-2250-439c-9e87-df8b2250339c::202202_147058_147550_344} system.asynchronous_metric_log (de87df8b-2250-439c-9e87-df8b2250339c) (MergerMutator): Merged 2 parts: from 202202_147058_147549_343 to 202202_147550_147550_0 -2022.02.02 14:49:32.508440 [ 200359 ] {} MemoryTracker: Peak memory usage Mutate/Merge: 16.31 MiB. -2022.02.02 14:49:33.000148 [ 200388 ] {} AsynchronousMetrics: MemoryTracking: was 774.16 MiB, peak 2.51 GiB, will set to 772.30 MiB (RSS), difference: -1.86 MiB diff --git a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz deleted file mode 100644 index 136bf5913aa..00000000000 --- a/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz +++ /dev/null @@ -1 +0,0 @@ -dummy hz file for tests From 758b07db3c9eebea354e1f20f6d14b4102d1d3d8 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 08:45:35 +0100 Subject: [PATCH 240/332] Update 02967_parallel_replicas_join_algo_and_analyzer #ci_set_analyzer --- ..._replicas_join_algo_and_analyzer.reference | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference index fa343571ba0..165504174d0 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference @@ -25,8 +25,8 @@ simple (global) join with analyzer and parallel replicas SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level = 'trace', max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level = 'trace', max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas @@ -40,8 +40,8 @@ simple (local) join with analyzer and parallel replicas 4200042 4200042 4200042 -1400014 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas and full sorting merge join @@ -55,8 +55,8 @@ simple (local) join with analyzer and parallel replicas and full sorting merge j 4200042 4200042 4200042 -1400014 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer @@ -82,8 +82,8 @@ nested join with analyzer and parallel replicas, both local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, join_algorithm = 'full_sorting_merge', send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both global @@ -97,11 +97,11 @@ nested join with analyzer and parallel replicas, both global 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, global + local @@ -115,11 +115,11 @@ nested join with analyzer and parallel replicas, global + local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS parallel_replicas_prefer_local_join = 1 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join = 0 (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting merge join @@ -133,11 +133,11 @@ nested join with analyzer and parallel replicas, both local, both full sorting m 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -151,11 +151,11 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'hash' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'hash' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -169,9 +169,9 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` SETTINGS join_algorithm = 'full_sorting_merge' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'hash' (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') SETTINGS allow_experimental_analyzer = 1, parallel_replicas_prefer_local_join = 0, send_logs_level = 'trace', allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm = 'hash' (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done From 4b144f94f8ef2f3fa3cc90fcf6431f4d75691189 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Mar 2024 14:24:55 +0300 Subject: [PATCH 241/332] MergeTree read split ranges into intersecting and non intersecting injection --- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + src/Processors/QueryPlan/PartsSplitter.cpp | 10 ++- .../QueryPlan/ReadFromMergeTree.cpp | 76 +++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5c25368fb00..b7ed8fc7d1d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -287,6 +287,7 @@ class IColumn; M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ \ M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ + M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 1.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ \ M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 0bce814170b..e8e38ca82f9 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -131,6 +131,7 @@ static std::map sett {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 5235785907c..33151597a38 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -34,6 +34,12 @@ std::string toString(const Values & value) return fmt::format("({})", fmt::join(value, ", ")); } +/** We rely that FieldVisitorAccurateLess will have strict weak ordering for any Field values including + * NaN, Null and containers (Array, Tuple, Map) that contain NaN or Null. But right now it does not properly + * support NaN and Nulls inside containers, because it uses Field operator< or accurate::lessOp for comparison + * that compares Nulls and NaNs differently than FieldVisitorAccurateLess. + * TODO: Update Field operator< to compare NaNs and Nulls the same way as FieldVisitorAccurateLess. + */ bool isSafePrimaryDataKeyType(const IDataType & data_type) { auto type_id = data_type.getTypeId(); @@ -316,12 +322,12 @@ struct PartRangeIndex bool operator==(const PartRangeIndex & other) const { - return part_index == other.part_index && range.begin == other.range.begin && range.end == other.range.end; + return std::tie(part_index, range.begin, range.end) == std::tie(other.part_index, other.range.begin, other.range.end); } bool operator<(const PartRangeIndex & other) const { - return part_index < other.part_index && range.begin < other.range.begin && range.end < other.range.end; + return std::tie(part_index, range.begin, range.end) < std::tie(other.part_index, other.range.begin, other.range.end); } size_t part_index; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f1ce5a7802f..4689e67a1ae 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -766,6 +766,82 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ auto read_type = is_parallel_reading_from_replicas ? ReadType::ParallelReplicas : ReadType::Default; + double read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = settings.merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability; + std::bernoulli_distribution fault(read_split_ranges_into_intersecting_and_non_intersecting_injection_probability); + + if (read_type != ReadType::ParallelReplicas && + num_streams > 1 && + read_split_ranges_into_intersecting_and_non_intersecting_injection_probability >= 0.0 && + fault(thread_local_rng) && + !isQueryWithFinal() && + data.merging_params.is_deleted_column.empty() && + !prewhere_info) + { + NameSet column_names_set(column_names.begin(), column_names.end()); + Names in_order_column_names_to_read(column_names); + + /// Add columns needed to calculate the sorting expression + for (const auto & column_name : metadata_for_reading->getColumnsRequiredForSortingKey()) + { + if (column_names_set.contains(column_name)) + continue; + + in_order_column_names_to_read.push_back(column_name); + column_names_set.insert(column_name); + } + + auto in_order_reading_step_getter = [this, &in_order_column_names_to_read, &info](auto parts) + { + return this->read( + std::move(parts), + in_order_column_names_to_read, + ReadType::InOrder, + 1 /* num_streams */, + 0 /* min_marks_for_concurrent_read */, + info.use_uncompressed_cache); + }; + + auto sorting_expr = std::make_shared(metadata_for_reading->getSortingKey().expression->getActionsDAG().clone()); + + SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey( + metadata_for_reading->getPrimaryKey(), + std::move(sorting_expr), + std::move(parts_with_ranges), + num_streams, + context, + std::move(in_order_reading_step_getter), + true /*split_parts_ranges_into_intersecting_and_non_intersecting_final*/, + true /*split_intersecting_parts_ranges_into_layers*/); + + auto merging_pipes = std::move(split_ranges_result.merging_pipes); + auto non_intersecting_parts_ranges_read_pipe = read(std::move(split_ranges_result.non_intersecting_parts_ranges), + column_names, + read_type, + num_streams, + info.min_marks_for_concurrent_read, + info.use_uncompressed_cache); + + if (merging_pipes.empty()) + return non_intersecting_parts_ranges_read_pipe; + + Pipes pipes; + pipes.resize(2); + pipes[0] = Pipe::unitePipes(std::move(merging_pipes)); + pipes[1] = std::move(non_intersecting_parts_ranges_read_pipe); + + auto conversion_action = ActionsDAG::makeConvertingActions( + pipes[0].getHeader().getColumnsWithTypeAndName(), + pipes[1].getHeader().getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + pipes[0].addSimpleTransform( + [conversion_action](const Block & header) + { + auto converting_expr = std::make_shared(conversion_action); + return std::make_shared(header, converting_expr); + }); + return Pipe::unitePipes(std::move(pipes)); + } + return read(std::move(parts_with_ranges), column_names, read_type, From a1435fc915a151ef057065854f8b8ed79fd7af2c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Mar 2024 14:27:12 +0300 Subject: [PATCH 242/332] Fixed tests --- .../00278_insert_already_sorted.sql | 4 +- .../0_stateless/00319_index_for_like.sql | 4 +- .../0_stateless/00612_pk_in_tuple_perf.sh | 4 +- .../0_stateless/00943_materialize_index.sh | 20 +++--- .../00944_clear_index_in_partition.sh | 12 ++-- .../0_stateless/00945_bloom_filter_index.sql | 1 + .../00974_primary_key_for_lowCardinality.sh | 4 +- ...ices_mutation_replicated_zookeeper_long.sh | 12 ++-- .../00979_toFloat_monotonicity.sql | 2 + ..._materialize_clear_index_compact_parts.sql | 2 + .../01200_mutations_memory_consumption.sql | 2 + .../0_stateless/01234_to_string_monotonic.sql | 2 + ...with_constant_string_in_index_analysis.sql | 2 + .../01585_use_index_for_global_in.sql | 2 + .../0_stateless/01624_soft_constraints.sh | 2 +- .../01710_aggregate_projections.sh | 2 +- .../0_stateless/01710_projection_in_index.sql | 2 + .../01748_partition_id_pruning.sql | 2 + .../02155_read_in_order_max_rows_to_read.sql | 2 + ...3_optimize_aggregation_in_order_prefix.sql | 2 + .../02340_parts_refcnt_mergetree.sh | 4 +- .../02343_aggregation_pipeline.sql | 1 + .../02346_inverted_index_search.sql | 61 ++++++++++--------- ...ting_by_input_stream_properties_explain.sh | 6 +- .../02402_merge_engine_with_view.sql | 2 + .../02418_tautological_if_index.sql | 2 + ...493_max_streams_for_merge_tree_reading.sql | 2 + .../02494_query_cache_nested_query_bug.sh | 8 ++- .../02521_aggregation_by_partitions.sql | 2 + .../0_stateless/02532_send_logs_level_test.sh | 2 +- .../02582_async_reading_with_small_limit.sql | 2 + .../02714_read_bytes_aggregateFunction.sql | 2 + .../02862_sorted_distinct_sparse_fix.sql | 2 + ...02899_indexing_by_space_filling_curves.sql | 2 + .../0_stateless/02949_ttl_group_by_bug.sql | 2 + .../02950_part_offset_as_primary_key.sql | 2 + 36 files changed, 118 insertions(+), 69 deletions(-) diff --git a/tests/queries/0_stateless/00278_insert_already_sorted.sql b/tests/queries/0_stateless/00278_insert_already_sorted.sql index b3de48dc155..dbd129d85cd 100644 --- a/tests/queries/0_stateless/00278_insert_already_sorted.sql +++ b/tests/queries/0_stateless/00278_insert_already_sorted.sql @@ -7,11 +7,11 @@ INSERT INTO sorted (x) SELECT intDiv(number, 100000) AS x FROM system.numbers LI SET max_threads = 1; SELECT count() FROM sorted; -SELECT DISTINCT x FROM sorted; +SELECT x FROM (SELECT DISTINCT x FROM sorted) ORDER BY x; INSERT INTO sorted (x) SELECT (intHash64(number) % 1000 = 0 ? 999 : intDiv(number, 100000)) AS x FROM system.numbers LIMIT 1000000; SELECT count() FROM sorted; -SELECT DISTINCT x FROM sorted; +SELECT x FROM (SELECT DISTINCT x FROM sorted) ORDER BY x; DROP TABLE sorted; diff --git a/tests/queries/0_stateless/00319_index_for_like.sql b/tests/queries/0_stateless/00319_index_for_like.sql index e490e595142..6abd1edc3ed 100644 --- a/tests/queries/0_stateless/00319_index_for_like.sql +++ b/tests/queries/0_stateless/00319_index_for_like.sql @@ -1,8 +1,10 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS index_for_like; set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE index_for_like (s String, d Date DEFAULT today()) ENGINE = MergeTree(d, (s, d), 1); -INSERT INTO index_for_like (s) VALUES ('Hello'), ('Hello, World'), ('Hello, World 1'), ('Hello 1'), ('Goodbye'), ('Goodbye, World'), ('Goodbye 1'), ('Goodbye, World 1'); +INSERT INTO index_for_like (s) VALUES ('Hello'), ('Hello, World'), ('Hello, World 1'), ('Hello 1'), ('Goodbye'), ('Goodbye, World'), ('Goodbye 1'), ('Goodbye, World 1'); SET max_rows_to_read = 3; SELECT s FROM index_for_like WHERE s LIKE 'Hello, World%'; diff --git a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh index 99813d894ae..c8297635c43 100755 --- a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh +++ b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh @@ -20,7 +20,7 @@ SETTINGS index_granularity = 1; INSERT INTO pk_in_tuple_perf SELECT number, number * 10 FROM numbers(100); EOF -query="SELECT count() FROM pk_in_tuple_perf WHERE (v, u) IN ((2, 10), (2, 20))" +query="SELECT count() FROM pk_in_tuple_perf WHERE (v, u) IN ((2, 10), (2, 20)) SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" $CLICKHOUSE_CLIENT --query "$query" $CLICKHOUSE_CLIENT --query "$query FORMAT JSON" | grep "rows_read" @@ -40,7 +40,7 @@ SETTINGS index_granularity = 1; INSERT INTO pk_in_tuple_perf_non_const SELECT today() - number, number FROM numbers(100); EOF -query="SELECT count() FROM pk_in_tuple_perf_non_const WHERE (u, d) IN ((0, today()), (1, today()))" +query="SELECT count() FROM pk_in_tuple_perf_non_const WHERE (u, d) IN ((0, today()), (1, today())) SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" $CLICKHOUSE_CLIENT --query "$query" $CLICKHOUSE_CLIENT --query "$query FORMAT JSON" | grep "rows_read" diff --git a/tests/queries/0_stateless/00943_materialize_index.sh b/tests/queries/0_stateless/00943_materialize_index.sh index 30ef46e5cb0..6ff7d34a9d7 100755 --- a/tests/queries/0_stateless/00943_materialize_index.sh +++ b/tests/queries/0_stateless/00943_materialize_index.sh @@ -31,31 +31,31 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES (8, 1, 2), (9, 1, 2)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT -n --query=" ALTER TABLE minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 SETTINGS mutations_sync = 2;" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 2 SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 2 SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx SETTINGS mutations_sync = 2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/00944_clear_index_in_partition.sh b/tests/queries/0_stateless/00944_clear_index_in_partition.sh index 8b74bd94f2c..4655077960f 100755 --- a/tests/queries/0_stateless/00944_clear_index_in_partition.sh +++ b/tests/queries/0_stateless/00944_clear_index_in_partition.sh @@ -32,17 +32,17 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES (8, 1, 2), (9, 1, 2)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 4 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 4 $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx CLEAR INDEX idx IN PARTITION 1;" --mutations_sync=1 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 6 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 6 $CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1;" --mutations_sync=1 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 FORMAT JSON" | grep "rows_read" # Returns 4 +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" # Returns 4 $CLICKHOUSE_CLIENT --query="DROP TABLE minmax_idx" diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql index faa7feda04d..4c26988574a 100644 --- a/tests/queries/0_stateless/00945_bloom_filter_index.sql +++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql @@ -1,4 +1,5 @@ SET allow_suspicious_low_cardinality_types=1; +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; DROP TABLE IF EXISTS single_column_bloom_filter; diff --git a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh index f8527cd491e..389d433c7e2 100755 --- a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh +++ b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh @@ -32,9 +32,9 @@ $CLICKHOUSE_CLIENT --query="insert into lowString (a, b) select top 100000 toStr $CLICKHOUSE_CLIENT --query="insert into string (a, b) select top 100000 toString(number), today() from system.numbers" -$CLICKHOUSE_CLIENT --query="select count() from lowString where a in ('1', '2') FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="select count() from lowString where a in ('1', '2') SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="select count() from string where a in ('1', '2') FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="select count() from string where a in ('1', '2') SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE lowString;" $CLICKHOUSE_CLIENT --query="DROP TABLE string;" diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh index 89b17ffe2bf..686dd7f6df0 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh @@ -49,18 +49,18 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO indices_mutaions1 VALUES $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA indices_mutaions2" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="ALTER TABLE indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2;" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" +$CLICKHOUSE_CLIENT --query="SELECT count() FROM indices_mutaions2 WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON;" | grep "rows_read" $CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions1" $CLICKHOUSE_CLIENT --query="DROP TABLE indices_mutaions2" diff --git a/tests/queries/0_stateless/00979_toFloat_monotonicity.sql b/tests/queries/0_stateless/00979_toFloat_monotonicity.sql index 9b3bdf91573..fa1f5b17a1c 100644 --- a/tests/queries/0_stateless/00979_toFloat_monotonicity.sql +++ b/tests/queries/0_stateless/00979_toFloat_monotonicity.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test1; DROP TABLE IF EXISTS test2; DROP TABLE IF EXISTS test3; diff --git a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql index 767ca0e4073..b2ebe7e2cc2 100644 --- a/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql +++ b/tests/queries/0_stateless/01114_materialize_clear_index_compact_parts.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS minmax_compact; CREATE TABLE minmax_compact diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index bca2286aa22..61263d125b0 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,5 +1,7 @@ -- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-merge-tree-settings +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS table_with_single_pk; CREATE TABLE table_with_single_pk diff --git a/tests/queries/0_stateless/01234_to_string_monotonic.sql b/tests/queries/0_stateless/01234_to_string_monotonic.sql index 87324fdda27..65b7ab21073 100644 --- a/tests/queries/0_stateless/01234_to_string_monotonic.sql +++ b/tests/queries/0_stateless/01234_to_string_monotonic.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test1; DROP TABLE IF EXISTS test2; diff --git a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql index 50aa434a28c..b7778dfd780 100644 --- a/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql +++ b/tests/queries/0_stateless/01312_comparison_with_constant_string_in_index_analysis.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test; CREATE TABLE test (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 1000, index_granularity_bytes = '10Mi'; INSERT INTO test SELECT * FROM numbers(1000000); diff --git a/tests/queries/0_stateless/01585_use_index_for_global_in.sql b/tests/queries/0_stateless/01585_use_index_for_global_in.sql index 1dd7609350f..3b0ca726dfa 100644 --- a/tests/queries/0_stateless/01585_use_index_for_global_in.sql +++ b/tests/queries/0_stateless/01585_use_index_for_global_in.sql @@ -1,5 +1,7 @@ -- Tags: global +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists xp; drop table if exists xp_d; diff --git a/tests/queries/0_stateless/01624_soft_constraints.sh b/tests/queries/0_stateless/01624_soft_constraints.sh index 944a4e4234f..aae193f6a5d 100755 --- a/tests/queries/0_stateless/01624_soft_constraints.sh +++ b/tests/queries/0_stateless/01624_soft_constraints.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; SET optimize_substitute_columns = 1; SET optimize_append_index = 1" +SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; SET optimize_substitute_columns = 1; SET optimize_append_index = 1; SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" $CLICKHOUSE_CLIENT -n --query=" $SETTINGS; diff --git a/tests/queries/0_stateless/01710_aggregate_projections.sh b/tests/queries/0_stateless/01710_aggregate_projections.sh index 7ea40365937..b02d961bf97 100755 --- a/tests/queries/0_stateless/01710_aggregate_projections.sh +++ b/tests/queries/0_stateless/01710_aggregate_projections.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh # Number of read rows depends on max_bytes_before_external_group_by. -CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0" +CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability 0.0" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_agg_proj (x Int32, y Int32, PROJECTION x_plus_y (SELECT sum(x - y), argMax(x, y) group by x + y)) ENGINE = MergeTree ORDER BY tuple() settings index_granularity = 1" $CLICKHOUSE_CLIENT -q "insert into test_agg_proj select intDiv(number, 2), -intDiv(number,3) - 1 from numbers(100)" diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql index b858418584c..425653ec501 100644 --- a/tests/queries/0_stateless/01710_projection_in_index.sql +++ b/tests/queries/0_stateless/01710_projection_in_index.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t (i int, j int, k int, projection p (select * order by j)) engine MergeTree order by i settings index_granularity = 1; diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index 9a26dd8daba..b637528bc6c 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists x; create table x (i int, j int) engine MergeTree partition by i order by j settings index_granularity = 1; diff --git a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql index b387582296d..4b47a860071 100644 --- a/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql +++ b/tests/queries/0_stateless/02155_read_in_order_max_rows_to_read.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS t_max_rows_to_read; CREATE TABLE t_max_rows_to_read (a UInt64) diff --git a/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql b/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql index 5065bd96bc1..8bc75040e5a 100644 --- a/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql +++ b/tests/queries/0_stateless/02233_optimize_aggregation_in_order_prefix.sql @@ -1,5 +1,7 @@ -- Tags: no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists data_02233; create table data_02233 (parent_key Int, child_key Int, value Int) engine=MergeTree() order by parent_key; diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index 9ea924377b2..208a9038681 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -15,11 +15,13 @@ function check_refcnt_for_table() local query_id query_id="$table-$(random_str 10)" + SETTINGS="--format Null --max_threads 1 --max_block_size 1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability 0.0" + # Notes: # - query may sleep 1*(200/4)=50 seconds maximum, it is enough to check system.parts # - "part = 1" condition should prune all parts except first # - max_block_size=1 with index_granularity=1 will allow to cancel the query earlier - $CLICKHOUSE_CLIENT --format Null --max_threads 1 --max_block_size 1 --query_id "$query_id" -q "select sleepEachRow(1) from $table where part = 1" & + $CLICKHOUSE_CLIENT $SETTINGS --query_id "$query_id" -q "select sleepEachRow(1) from $table where part = 1" & PID=$! # wait for query to be started diff --git a/tests/queries/0_stateless/02343_aggregation_pipeline.sql b/tests/queries/0_stateless/02343_aggregation_pipeline.sql index 496379f5c48..d73ac66763e 100644 --- a/tests/queries/0_stateless/02343_aggregation_pipeline.sql +++ b/tests/queries/0_stateless/02343_aggregation_pipeline.sql @@ -2,6 +2,7 @@ -- produces different pipeline if enabled set enable_memory_bound_merging_of_aggregation_results = 0; +set merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; set max_threads = 16; set prefer_localhost_replica = 1; diff --git a/tests/queries/0_stateless/02346_inverted_index_search.sql b/tests/queries/0_stateless/02346_inverted_index_search.sql index d225d3463d1..052703dceaf 100644 --- a/tests/queries/0_stateless/02346_inverted_index_search.sql +++ b/tests/queries/0_stateless/02346_inverted_index_search.sql @@ -1,5 +1,6 @@ SET allow_experimental_inverted_index = 1; SET log_queries = 1; +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; ---------------------------------------------------- SELECT 'Test inverted(2)'; @@ -23,7 +24,7 @@ SELECT * FROM tab WHERE s == 'Alick a01'; -- check the query only read 1 granules (2 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log +SELECT read_rows==2 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s == \'Alick a01\';') @@ -36,7 +37,7 @@ SELECT * FROM tab WHERE s LIKE '%01%' ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%01%\' ORDER BY k;') @@ -49,11 +50,11 @@ SELECT * FROM tab WHERE hasToken(s, 'Click') ORDER BY k; -- check the query only read 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE hasToken(s, \'Click\') ORDER BY k;') - AND type='QueryFinish' + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -76,11 +77,11 @@ SELECT * FROM tab_x WHERE hasToken(s, 'Alick') ORDER BY k; -- check the query only read 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE hasToken(s, \'Alick\');') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE hasToken(s, \'Alick\');') + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -89,24 +90,24 @@ SELECT * FROM tab_x WHERE s IN ('Alick a01', 'Alick a06') ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE s IN (\'Alick a01\', \'Alick a06\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE s IN (\'Alick a01\', \'Alick a06\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==2 LIMIT 1; --- search inverted index with multiSearch +-- search inverted index with multiSearch SELECT * FROM tab_x WHERE multiSearchAny(s, ['a01', 'b01']) ORDER BY k; -- check the query only read 2 granules (4 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==4 from system.query_log +SELECT read_rows==4 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE multiSearchAny(s, [\'a01\', \'b01\']) ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab_x WHERE multiSearchAny(s, [\'a01\', \'b01\']) ORDER BY k;') + AND type='QueryFinish' AND result_rows==2 LIMIT 1; @@ -129,11 +130,11 @@ SELECT * FROM tab WHERE has(s, 'Click a03') ORDER BY k; -- check the query must read all 10 granules (20 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log +SELECT read_rows==2 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE has(s, \'Click a03\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE has(s, \'Click a03\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; @@ -156,11 +157,11 @@ SELECT * FROM tab WHERE mapContains(s, 'Click') ORDER BY k; -- check the query must read all 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE mapContains(s, \'Click\') ORDER BY k;') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE mapContains(s, \'Click\') ORDER BY k;') + AND type='QueryFinish' AND result_rows==4 LIMIT 1; @@ -169,11 +170,11 @@ SELECT * FROM tab WHERE s['Click'] = 'Click a03'; -- check the query must read all 4 granules (8 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==8 from system.query_log +SELECT read_rows==8 from system.query_log WHERE query_kind ='Select' AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s[\'Click\'] = \'Click a03\';') - AND type='QueryFinish' + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s[\'Click\'] = \'Click a03\';') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; @@ -199,10 +200,10 @@ SELECT * FROM tab WHERE s LIKE '%01%' ORDER BY k; -- check the query only read 3 granules (6 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; SELECT read_rows==6 from system.query_log - WHERE query_kind ='Select' + WHERE query_kind ='Select' AND current_database = currentDatabase() AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%01%\' ORDER BY k;') - AND type='QueryFinish' + AND type='QueryFinish' AND result_rows==3 LIMIT 1; @@ -226,11 +227,11 @@ SELECT * FROM tab WHERE s LIKE '%你好%' ORDER BY k; -- check the query only read 1 granule (2 rows total; each granule has 2 rows) SYSTEM FLUSH LOGS; -SELECT read_rows==2 from system.query_log - WHERE query_kind ='Select' - AND current_database = currentDatabase() - AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%你好%\' ORDER BY k;') - AND type='QueryFinish' +SELECT read_rows==2 from system.query_log + WHERE query_kind ='Select' + AND current_database = currentDatabase() + AND endsWith(trimRight(query), 'SELECT * FROM tab WHERE s LIKE \'%你好%\' ORDER BY k;') + AND type='QueryFinish' AND result_rows==1 LIMIT 1; diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh index 7e937ac42b6..c223fcc86bc 100755 --- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh +++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh @@ -15,14 +15,14 @@ FIND_SORTMODE="$GREP_SORTMODE | $TRIM_LEADING_SPACES" function explain_sorting { echo "-- QUERY: "$1 - $CLICKHOUSE_CLIENT -nq "$1" | eval $FIND_SORTING + $CLICKHOUSE_CLIENT --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTING } function explain_sortmode { echo "-- QUERY: "$1 - $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -nq "$1" | eval $FIND_SORTMODE + $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE echo "-- QUERY (analyzer): "$1 - $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -nq "$1" | eval $FIND_SORTMODE + $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE } $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync" diff --git a/tests/queries/0_stateless/02402_merge_engine_with_view.sql b/tests/queries/0_stateless/02402_merge_engine_with_view.sql index 81c2d67d05b..3998f410a63 100644 --- a/tests/queries/0_stateless/02402_merge_engine_with_view.sql +++ b/tests/queries/0_stateless/02402_merge_engine_with_view.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + -- #40014 CREATE TABLE m0 (id UInt64) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 1, ratio_of_defaults_for_sparse_serialization = 1.0; INSERT INTO m0 SELECT number FROM numbers(10); diff --git a/tests/queries/0_stateless/02418_tautological_if_index.sql b/tests/queries/0_stateless/02418_tautological_if_index.sql index c37f24b29f6..1368347e88f 100644 --- a/tests/queries/0_stateless/02418_tautological_if_index.sql +++ b/tests/queries/0_stateless/02418_tautological_if_index.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS constCondOptimization; CREATE TABLE constCondOptimization diff --git a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql index f2e81273f12..cbf645ddec5 100644 --- a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql +++ b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql @@ -1,5 +1,7 @@ -- Tags: no-random-merge-tree-settings +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t (x UInt64) engine = MergeTree order by x; insert into t select number from numbers_mt(10000000) settings max_insert_threads=8; diff --git a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh index 394367919e9..8712c7c84c6 100755 --- a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh +++ b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh @@ -15,9 +15,11 @@ ${CLICKHOUSE_CLIENT} --query "CREATE TABLE tab (a UInt64) ENGINE=MergeTree() ORD ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (1) (2) (3)" ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (3) (4) (5)" +SETTINGS="SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0" + # Verify that the first query does two aggregations and the second query zero aggregations. Since query cache is currently not integrated -# with EXPLAIN PLAN, we need need to check the logs. -${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0" 2>&1 | grep "Aggregated. " | wc -l -${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0" 2>&1 | grep "Aggregated. " | wc -l +# with EXPLAIN PLAN, we need to check the logs. +${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab $SETTINGS" 2>&1 | grep "Aggregated. " | wc -l +${CLICKHOUSE_CLIENT} --send_logs_level=trace --query "SELECT count(a) / (SELECT sum(a) FROM tab) FROM tab $SETTINGS" 2>&1 | grep "Aggregated. " | wc -l ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP QUERY CACHE" diff --git a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql index 5b013ca5aef..a90e56a9822 100644 --- a/tests/queries/0_stateless/02521_aggregation_by_partitions.sql +++ b/tests/queries/0_stateless/02521_aggregation_by_partitions.sql @@ -1,5 +1,7 @@ -- Tags: long, no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + set max_threads = 16; set allow_aggregate_partitions_independently = 1; set force_aggregate_partitions_independently = 1; diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.sh b/tests/queries/0_stateless/02532_send_logs_level_test.sh index 7dd9a152385..f65d8705569 100755 --- a/tests/queries/0_stateless/02532_send_logs_level_test.sh +++ b/tests/queries/0_stateless/02532_send_logs_level_test.sh @@ -17,6 +17,6 @@ $CLICKHOUSE_CLIENT -nm -q " # instead of "last" value, hence you cannot simply append another # --send_logs_level here. CLICKHOUSE_CLIENT_CLEAN=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=test/g') -$CLICKHOUSE_CLIENT_CLEAN -q "select * from data" |& grep -o -e '.*' -e '.*' +$CLICKHOUSE_CLIENT_CLEAN -q "select * from data SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" |& grep -o -e '.*' -e '.*' $CLICKHOUSE_CLIENT -q "drop table data" diff --git a/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql index e51666673b1..cb6b1b6083e 100644 --- a/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql +++ b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql @@ -1,5 +1,7 @@ -- Tags: no-s3-storage +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists t; create table t(a UInt64) engine=MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql index 1c70a77c4d1..eb0a37045ef 100644 --- a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql +++ b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + CREATE TABLE test (id UInt64, `amax` AggregateFunction(argMax, String, DateTime)) ENGINE=MergeTree() ORDER BY id diff --git a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql index 2bcdb3d43ff..d303a53fc39 100644 --- a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql +++ b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS t_sparse_distinct; CREATE TABLE t_sparse_distinct (id UInt32, v String) diff --git a/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql b/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql index c7325b2478d..a3989039f50 100644 --- a/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql +++ b/tests/queries/0_stateless/02899_indexing_by_space_filling_curves.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test; CREATE TABLE test (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY mortonEncode(x, y) SETTINGS index_granularity = 8192, index_granularity_bytes = '1Mi'; diff --git a/tests/queries/0_stateless/02949_ttl_group_by_bug.sql b/tests/queries/0_stateless/02949_ttl_group_by_bug.sql index a3d0794c897..83776cc9d85 100644 --- a/tests/queries/0_stateless/02949_ttl_group_by_bug.sql +++ b/tests/queries/0_stateless/02949_ttl_group_by_bug.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS ttl_group_by_bug; CREATE TABLE ttl_group_by_bug diff --git a/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql b/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql index 736d54023ce..31a82cce1cd 100644 --- a/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql +++ b/tests/queries/0_stateless/02950_part_offset_as_primary_key.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + drop table if exists a; create table a (i int) engine MergeTree order by i settings index_granularity = 2; From b8c53d7cf3f8de58ca8778be4bb87df4faaa6cca Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 12:06:19 +0300 Subject: [PATCH 243/332] Fixed tests --- src/Core/Settings.h | 2 +- src/Processors/QueryPlan/PartsSplitter.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- tests/ci/stress.py | 3 +++ tests/clickhouse-test | 1 + .../0_stateless/01200_mutations_memory_consumption.sql | 4 +--- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b7ed8fc7d1d..891bd8c0de9 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -287,7 +287,7 @@ class IColumn; M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ \ M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ - M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 1.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ + M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ \ M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 33151597a38..2af1bcb0260 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -792,7 +792,7 @@ ASTs buildFilters(const KeyDescription & primary_key, const std::vector const auto & type = primary_key.data_types.at(i); // PK may contain functions of the table columns, so we need the actual PK AST with all expressions it contains. - auto pk_ast = primary_key.expression_list_ast->children.at(i); + auto pk_ast = primary_key.expression_list_ast->children.at(i)->clone(); // If PK is nullable, prepend a null mask column for > comparison. // Also transform the AST into assumeNotNull(pk) so that the result type is not-nullable. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4689e67a1ae..f4607cad040 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -771,7 +771,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ if (read_type != ReadType::ParallelReplicas && num_streams > 1 && - read_split_ranges_into_intersecting_and_non_intersecting_injection_probability >= 0.0 && + read_split_ranges_into_intersecting_and_non_intersecting_injection_probability > 0.0 && fault(thread_local_rng) && !isQueryWithFinal() && data.merging_params.is_deleted_column.empty() && diff --git a/tests/ci/stress.py b/tests/ci/stress.py index 7ccc058f79f..dbc04835fae 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -52,6 +52,9 @@ def get_options(i: int, upgrade_check: bool) -> str: if i % 5 == 1: client_options.append("memory_tracker_fault_probability=0.001") + if i % 5 == 1: + client_options.append("merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05") + if i % 2 == 1 and not upgrade_check: client_options.append("group_by_use_nulls=1") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 13d1200758e..6eac6e537ec 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -843,6 +843,7 @@ class SettingsRandomizer: "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10), "use_page_cache_for_disks_without_file_cache": lambda: random.random() < 0.7, "page_cache_inject_eviction": lambda: random.random() < 0.5, + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda : round(random.random(), 2) } @staticmethod diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index 61263d125b0..2266da5fc8f 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,6 +1,4 @@ --- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-merge-tree-settings - -SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; +-- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-settings, no-random-merge-tree-settings DROP TABLE IF EXISTS table_with_single_pk; From 4dc985aa2fc9dcb0ded21e70e175ca41cd25e785 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 12:20:51 +0300 Subject: [PATCH 244/332] Fixed style check --- tests/ci/stress.py | 4 +++- tests/clickhouse-test | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index dbc04835fae..e0601b86f00 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -53,7 +53,9 @@ def get_options(i: int, upgrade_check: bool) -> str: client_options.append("memory_tracker_fault_probability=0.001") if i % 5 == 1: - client_options.append("merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05") + client_options.append( + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.05" + ) if i % 2 == 1 and not upgrade_check: client_options.append("group_by_use_nulls=1") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 6eac6e537ec..0dd408a2c8c 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -843,7 +843,9 @@ class SettingsRandomizer: "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10), "use_page_cache_for_disks_without_file_cache": lambda: random.random() < 0.7, "page_cache_inject_eviction": lambda: random.random() < 0.5, - "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda : round(random.random(), 2) + "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda: round( + random.random(), 2 + ), } @staticmethod From 218655228887bed057c057fd89afbffef0b9681c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 6 Mar 2024 18:41:56 +0300 Subject: [PATCH 245/332] Updated SettingsChangesHistory --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index e8e38ca82f9..be31dd52b10 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -102,6 +102,7 @@ static std::map sett {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, + {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, @@ -131,7 +132,6 @@ static std::map sett {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, }}, {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, From d8a31160b4547e9e56eeb4af768272faacc35bf2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 11 Mar 2024 11:20:54 +0300 Subject: [PATCH 246/332] Fixed tests --- tests/queries/0_stateless/02302_s3_file_pruning.sql | 2 ++ tests/queries/0_stateless/02987_group_array_intersect.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.sql b/tests/queries/0_stateless/02302_s3_file_pruning.sql index 93fc8a1bc25..647dfd5e5eb 100644 --- a/tests/queries/0_stateless/02302_s3_file_pruning.sql +++ b/tests/queries/0_stateless/02302_s3_file_pruning.sql @@ -1,6 +1,8 @@ -- Tags: no-parallel, no-fasttest -- Tag no-fasttest: Depends on S3 +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + -- { echo } drop table if exists test_02302; create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; diff --git a/tests/queries/0_stateless/02987_group_array_intersect.sql b/tests/queries/0_stateless/02987_group_array_intersect.sql index 703914e464d..321e860b0a8 100644 --- a/tests/queries/0_stateless/02987_group_array_intersect.sql +++ b/tests/queries/0_stateless/02987_group_array_intersect.sql @@ -1,3 +1,5 @@ +SET merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0; + DROP TABLE IF EXISTS test_empty; CREATE TABLE test_empty (a Array(Int64)) engine=MergeTree ORDER BY a; INSERT INTO test_empty VALUES ([]); From a5d286dc27ca3d9d32217be3c4d7fdd55ec82b0c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 13 Mar 2024 18:13:34 +0300 Subject: [PATCH 247/332] Fixed tests --- tests/queries/0_stateless/01508_partition_pruning_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh index c1f2d6562ab..b951e550708 100755 --- a/tests/queries/0_stateless/01508_partition_pruning_long.sh +++ b/tests/queries/0_stateless/01508_partition_pruning_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-polymorphic-parts, no-random-merge-tree-settings +# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings # Description of test result: # Test the correctness of the partition pruning From f162ea83417ab93a6883ad5f3ac9daa80e013dcd Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 21 Mar 2024 09:51:10 +0000 Subject: [PATCH 248/332] Fix use-of-uninitialized-value in parseDateTimeBestEffort --- src/IO/parseDateTimeBestEffort.cpp | 3 +++ tests/queries/0_stateless/03014_msan_parse_date_time.reference | 0 tests/queries/0_stateless/03014_msan_parse_date_time.sql | 1 + 3 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/03014_msan_parse_date_time.reference create mode 100644 tests/queries/0_stateless/03014_msan_parse_date_time.sql diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index caf51d94bb3..83928b32f2f 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -147,6 +147,9 @@ ReturnType parseDateTimeBestEffortImpl( { has_comma_between_date_and_time = true; ++in.position(); + + if (in.eof()) + break; } } diff --git a/tests/queries/0_stateless/03014_msan_parse_date_time.reference b/tests/queries/0_stateless/03014_msan_parse_date_time.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03014_msan_parse_date_time.sql b/tests/queries/0_stateless/03014_msan_parse_date_time.sql new file mode 100644 index 00000000000..d6daea69cfa --- /dev/null +++ b/tests/queries/0_stateless/03014_msan_parse_date_time.sql @@ -0,0 +1 @@ +SELECT parseDateTimeBestEffort(toFixedString('01/12/2017,', 11)); -- { serverError CANNOT_PARSE_DATETIME } From 6e260d9419eaea891d972cf37447872980479531 Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:36 +0000 Subject: [PATCH 249/332] poco foundation: add illumos support --- base/poco/Foundation/src/Environment_UNIX.cpp | 6 +++--- base/poco/Foundation/src/NamedEvent_UNIX.cpp | 2 +- base/poco/Foundation/src/NamedMutex_UNIX.cpp | 2 +- base/poco/Net/CMakeLists.txt | 4 ++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/base/poco/Foundation/src/Environment_UNIX.cpp b/base/poco/Foundation/src/Environment_UNIX.cpp index 202e5d88f83..faabb374778 100644 --- a/base/poco/Foundation/src/Environment_UNIX.cpp +++ b/base/poco/Foundation/src/Environment_UNIX.cpp @@ -281,15 +281,15 @@ void EnvironmentImpl::nodeIdImpl(NodeId& id) /// #include #if defined(sun) || defined(__sun) #include +#include +#include +#include #endif /// #include /// #include /// #include /// #include /// #include -/// #include -/// #include -/// #include /// #include diff --git a/base/poco/Foundation/src/NamedEvent_UNIX.cpp b/base/poco/Foundation/src/NamedEvent_UNIX.cpp index 978e6e0bc02..3cda4104c73 100644 --- a/base/poco/Foundation/src/NamedEvent_UNIX.cpp +++ b/base/poco/Foundation/src/NamedEvent_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Foundation/src/NamedMutex_UNIX.cpp b/base/poco/Foundation/src/NamedMutex_UNIX.cpp index 6cfa1369c9d..d53d54d7bb5 100644 --- a/base/poco/Foundation/src/NamedMutex_UNIX.cpp +++ b/base/poco/Foundation/src/NamedMutex_UNIX.cpp @@ -31,7 +31,7 @@ namespace Poco { -#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) +#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS) union semun { int val; diff --git a/base/poco/Net/CMakeLists.txt b/base/poco/Net/CMakeLists.txt index 792045c9b43..50ffbdf905a 100644 --- a/base/poco/Net/CMakeLists.txt +++ b/base/poco/Net/CMakeLists.txt @@ -9,6 +9,10 @@ elseif (OS_DARWIN OR OS_FREEBSD) target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL) endif () +if (OS_SUNOS) + target_link_libraries (_poco_net PUBLIC socket nsl) +endif () + # TODO: remove these warning exclusions target_compile_options (_poco_net PRIVATE From 7abdc66e0d8f3be7f488cdcf9362f4b599f2a8ce Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Wed, 22 Nov 2023 11:47:33 +0000 Subject: [PATCH 250/332] contrib/c-ares: add illumos as a platform --- contrib/c-ares-cmake/CMakeLists.txt | 2 + contrib/c-ares-cmake/solaris/ares_build.h | 104 +++++ contrib/c-ares-cmake/solaris/ares_config.h | 503 +++++++++++++++++++++ 3 files changed, 609 insertions(+) create mode 100644 contrib/c-ares-cmake/solaris/ares_build.h create mode 100644 contrib/c-ares-cmake/solaris/ares_config.h diff --git a/contrib/c-ares-cmake/CMakeLists.txt b/contrib/c-ares-cmake/CMakeLists.txt index 86ab6f90260..daec96ff1b1 100644 --- a/contrib/c-ares-cmake/CMakeLists.txt +++ b/contrib/c-ares-cmake/CMakeLists.txt @@ -86,6 +86,8 @@ elseif (OS_DARWIN) target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE) elseif (OS_FREEBSD) target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd") +elseif (OS_SUNOS) + target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/solaris") endif() add_library(ch_contrib::c-ares ALIAS _c-ares) diff --git a/contrib/c-ares-cmake/solaris/ares_build.h b/contrib/c-ares-cmake/solaris/ares_build.h new file mode 100644 index 00000000000..f42b59d07bd --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_build.h @@ -0,0 +1,104 @@ +/* include/ares_build.h. Generated from ares_build.h.in by configure. */ +#ifndef __CARES_BUILD_H +#define __CARES_BUILD_H + + +/* Copyright (C) 2009 - 2021 by Daniel Stenberg et al + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose and without fee is hereby granted, provided + * that the above copyright notice appear in all copies and that both that + * copyright notice and this permission notice appear in supporting + * documentation, and that the name of M.I.T. not be used in advertising or + * publicity pertaining to distribution of the software without specific, + * written prior permission. M.I.T. makes no representations about the + * suitability of this software for any purpose. It is provided "as is" + * without express or implied warranty. + */ + +/* ================================================================ */ +/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * Nothing in this file is intended to be modified or adjusted by the + * c-ares library user nor by the c-ares library builder. + * + * If you think that something actually needs to be changed, adjusted + * or fixed in this file, then, report it on the c-ares development + * mailing list: http://lists.haxx.se/listinfo/c-ares/ + * + * This header file shall only export symbols which are 'cares' or 'CARES' + * prefixed, otherwise public name space would be polluted. + * + * NOTE 2: + * ------- + * + * Right now you might be staring at file ares_build.h.in or ares_build.h, + * this is due to the following reason: + * + * On systems capable of running the configure script, the configure process + * will overwrite the distributed ares_build.h file with one that is suitable + * and specific to the library being configured and built, which is generated + * from the ares_build.h.in template file. + * + */ + +/* ================================================================ */ +/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ +/* ================================================================ */ + +#ifdef CARES_TYPEOF_ARES_SOCKLEN_T +# error "CARES_TYPEOF_ARES_SOCKLEN_T shall not be defined except in ares_build.h" + Error Compilation_aborted_CARES_TYPEOF_ARES_SOCKLEN_T_already_defined +#endif + +#define CARES_HAVE_ARPA_NAMESER_H 1 +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* ================================================================ */ +/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ +/* ================================================================ */ + +/* Configure process defines this to 1 when it finds out that system */ +/* header file ws2tcpip.h must be included by the external interface. */ +/* #undef CARES_PULL_WS2TCPIP_H */ +#ifdef CARES_PULL_WS2TCPIP_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/types.h must be included by the external interface. */ +#define CARES_PULL_SYS_TYPES_H 1 +#ifdef CARES_PULL_SYS_TYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/socket.h must be included by the external interface. */ +#define CARES_PULL_SYS_SOCKET_H 1 +#ifdef CARES_PULL_SYS_SOCKET_H +# include +#endif + +/* Integral data type used for ares_socklen_t. */ +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t + +/* Data type definition of ares_socklen_t. */ +typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t; + +/* Integral data type used for ares_ssize_t. */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Data type definition of ares_ssize_t. */ +typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t; + +#endif /* __CARES_BUILD_H */ diff --git a/contrib/c-ares-cmake/solaris/ares_config.h b/contrib/c-ares-cmake/solaris/ares_config.h new file mode 100644 index 00000000000..c4ac5e38966 --- /dev/null +++ b/contrib/c-ares-cmake/solaris/ares_config.h @@ -0,0 +1,503 @@ +/* src/lib/ares_config.h. Generated from ares_config.h.in by configure. */ +/* src/lib/ares_config.h.in. Generated from configure.ac by autoheader. */ + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define this if ares is built for a big endian system */ +/* #undef ARES_BIG_ENDIAN */ + +/* Defined for build that exposes internal static functions for testing. */ +/* #undef CARES_EXPOSE_STATICS */ + +/* a suitable file/device to read random data from */ +#define CARES_RANDOM_FILE "/dev/urandom" + +/* Defined for build with symbol hiding. */ +#define CARES_SYMBOL_HIDING 1 + +/* Definition to make a library symbol externally visible. */ +#define CARES_SYMBOL_SCOPE_EXTERN __attribute__ ((__visibility__ ("default"))) + +/* the signed version of size_t */ +#define CARES_TYPEOF_ARES_SSIZE_T ssize_t + +/* Use resolver library to configure cares */ +/* #undef CARES_USE_LIBRESOLV */ + +/* if a /etc/inet dir is being used */ +#define ETC_INET 1 + +/* Define to the type of arg 2 for gethostname. */ +#define GETHOSTNAME_TYPE_ARG2 int + +/* Define to the type qualifier of arg 1 for getnameinfo. */ +#define GETNAMEINFO_QUAL_ARG1 const + +/* Define to the type of arg 1 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG1 struct sockaddr * + +/* Define to the type of arg 2 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG2 socklen_t + +/* Define to the type of args 4 and 6 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG46 socklen_t + +/* Define to the type of arg 7 for getnameinfo. */ +#define GETNAMEINFO_TYPE_ARG7 int + +/* Specifies the number of arguments to getservbyport_r */ +#define GETSERVBYPORT_R_ARGS 5 + +/* Specifies the size of the buffer to pass to getservbyport_r */ +#define GETSERVBYPORT_R_BUFSIZE 4096 + +/* Define to 1 if you have AF_INET6. */ +#define HAVE_AF_INET6 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_COMPAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bitncmp' function. */ +/* #undef HAVE_BITNCMP */ + +/* Define to 1 if bool is an available type. */ +#define HAVE_BOOL_T 1 + +/* Define to 1 if you have the clock_gettime function and monotonic timer. */ +#define HAVE_CLOCK_GETTIME_MONOTONIC 1 + +/* Define to 1 if you have the closesocket function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the CloseSocket camel case function. */ +/* #undef HAVE_CLOSESOCKET_CAMEL */ + +/* Define to 1 if you have the connect function. */ +#define HAVE_CONNECT 1 + +/* define if the compiler supports basic C++11 syntax */ +#define HAVE_CXX11 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the fcntl function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have a working fcntl O_NONBLOCK function. */ +#define HAVE_FCNTL_O_NONBLOCK 1 + +/* Define to 1 if you have the freeaddrinfo function. */ +#define HAVE_FREEADDRINFO 1 + +/* Define to 1 if you have a working getaddrinfo function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if the getaddrinfo function is threadsafe. */ +#define HAVE_GETADDRINFO_THREADSAFE 1 + +/* Define to 1 if you have the getenv function. */ +#define HAVE_GETENV 1 + +/* Define to 1 if you have the gethostbyaddr function. */ +#define HAVE_GETHOSTBYADDR 1 + +/* Define to 1 if you have the gethostbyname function. */ +#define HAVE_GETHOSTBYNAME 1 + +/* Define to 1 if you have the gethostname function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the getnameinfo function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the getservbyport_r function. */ +#define HAVE_GETSERVBYPORT_R 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `if_indextoname' function. */ +#define HAVE_IF_INDEXTONAME 1 + +/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */ +/* #undef HAVE_INET_NET_PTON */ + +/* Define to 1 if you have a IPv6 capable working inet_ntop function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have a IPv6 capable working inet_pton function. */ +#define HAVE_INET_PTON 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the ioctl function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the ioctlsocket function. */ +/* #undef HAVE_IOCTLSOCKET */ + +/* Define to 1 if you have the IoctlSocket camel case function. */ +/* #undef HAVE_IOCTLSOCKET_CAMEL */ + +/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function. + */ +/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */ + +/* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* #undef HAVE_IOCTLSOCKET_FIONBIO */ + +/* Define to 1 if you have a working ioctl FIONBIO function. */ +/* #undef HAVE_IOCTL_FIONBIO */ + +/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */ +/* #undef HAVE_IOCTL_SIOCGIFADDR */ + +/* Define to 1 if you have the `resolve' library (-lresolve). */ +/* #undef HAVE_LIBRESOLVE */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if your compiler supports LL */ +#define HAVE_LL 1 + +/* Define to 1 if the compiler supports the 'long long' data type. */ +#define HAVE_LONGLONG 1 + +/* Define to 1 if you have the malloc.h header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the memory.h header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the MSG_NOSIGNAL flag. */ +#define HAVE_MSG_NOSIGNAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H 1 + +/* Define to 1 if you have PF_INET6. */ +#define HAVE_PF_INET6 1 + +/* Define to 1 if you have the recv function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the recvfrom function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the send function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the setsockopt function. */ +#define HAVE_SETSOCKOPT 1 + +/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */ +/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if sig_atomic_t is an available typedef. */ +#define HAVE_SIG_ATOMIC_T 1 + +/* Define to 1 if sig_atomic_t is already defined as volatile. */ +/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */ + +/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */ +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 + +/* Define to 1 if you have the socket function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SOCKET_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDBOOL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the strcasecmp function. */ +#define HAVE_STRCASECMP 1 + +/* Define to 1 if you have the strcmpi function. */ +/* #undef HAVE_STRCMPI */ + +/* Define to 1 if you have the strdup function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the stricmp function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the strncasecmp function. */ +#define HAVE_STRNCASECMP 1 + +/* Define to 1 if you have the strncmpi function. */ +/* #undef HAVE_STRNCMPI */ + +/* Define to 1 if you have the strnicmp function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STROPTS_H 1 + +/* Define to 1 if you have struct addrinfo. */ +#define HAVE_STRUCT_ADDRINFO 1 + +/* Define to 1 if you have struct in6_addr. */ +#define HAVE_STRUCT_IN6_ADDR 1 + +/* Define to 1 if you have struct sockaddr_in6. */ +#define HAVE_STRUCT_SOCKADDR_IN6 1 + +/* if struct sockaddr_storage is defined */ +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 + +/* Define to 1 if you have the timeval struct. */ +#define HAVE_STRUCT_TIMEVAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the windows.h header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the winsock2.h header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the winsock.h header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the writev function. */ +#define HAVE_WRITEV 1 + +/* Define to 1 if you have the ws2tcpip.h header file. */ +/* #undef HAVE_WS2TCPIP_H */ + +/* Define if __system_property_get exists. */ +/* #undef HAVE___SYSTEM_PROPERTY_GET */ + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to 1 if you need the malloc.h header file even with stdlib.h */ +/* #undef NEED_MALLOC_H */ + +/* Define to 1 if you need the memory.h header file even with stdlib.h */ +/* #undef NEED_MEMORY_H */ + +/* Define to 1 if _REENTRANT preprocessor symbol must be defined. */ +#define NEED_REENTRANT 1 + +/* Define to 1 if _THREAD_SAFE preprocessor symbol must be defined. */ +/* #undef NEED_THREAD_SAFE */ + +/* cpu-machine-OS */ +#define OS "x86_64-pc-solaris2.11" + +/* Name of package */ +#define PACKAGE "c-ares" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "c-ares mailing list: http://lists.haxx.se/listinfo/c-ares" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "c-ares" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "c-ares 1.18.1" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "c-ares" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.18.1" + +/* Define to the type qualifier pointed by arg 5 for recvfrom. */ +#define RECVFROM_QUAL_ARG5 + +/* Define to the type of arg 1 for recvfrom. */ +#define RECVFROM_TYPE_ARG1 int + +/* Define to the type pointed by arg 2 for recvfrom. */ +#define RECVFROM_TYPE_ARG2 void + +/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG2_IS_VOID 1 + +/* Define to the type of arg 3 for recvfrom. */ +#define RECVFROM_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recvfrom. */ +#define RECVFROM_TYPE_ARG4 int + +/* Define to the type pointed by arg 5 for recvfrom. */ +#define RECVFROM_TYPE_ARG5 struct sockaddr + +/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */ +/* #undef RECVFROM_TYPE_ARG5_IS_VOID */ + +/* Define to the type pointed by arg 6 for recvfrom. */ +#define RECVFROM_TYPE_ARG6 void + +/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */ +#define RECVFROM_TYPE_ARG6_IS_VOID 1 + +/* Define to the function return type for recvfrom. */ +#define RECVFROM_TYPE_RETV ssize_t + +/* Define to the type of arg 1 for recv. */ +#define RECV_TYPE_ARG1 int + +/* Define to the type of arg 2 for recv. */ +#define RECV_TYPE_ARG2 void * + +/* Define to the type of arg 3 for recv. */ +#define RECV_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for recv. */ +#define RECV_TYPE_ARG4 int + +/* Define to the function return type for recv. */ +#define RECV_TYPE_RETV ssize_t + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type qualifier of arg 2 for send. */ +#define SEND_QUAL_ARG2 const + +/* Define to the type of arg 1 for send. */ +#define SEND_TYPE_ARG1 int + +/* Define to the type of arg 2 for send. */ +#define SEND_TYPE_ARG2 void * + +/* Define to the type of arg 3 for send. */ +#define SEND_TYPE_ARG3 size_t + +/* Define to the type of arg 4 for send. */ +#define SEND_TYPE_ARG4 int + +/* Define to the function return type for send. */ +#define SEND_TYPE_RETV ssize_t + +/* Define to 1 if all of the C90 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . This + macro is obsolete. */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to disable non-blocking sockets. */ +/* #undef USE_BLOCKING_SOCKETS */ + +/* Version number of package */ +#define VERSION "1.18.1" + +/* Define to avoid automatic inclusion of winsock.h */ +/* #undef WIN32_LEAN_AND_MEAN */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to 1 if OS is AIX. */ +#ifndef _ALL_SOURCE +/* # undef _ALL_SOURCE */ +#endif + +/* Number of bits in a file offset, on hosts where this is settable. */ +/* #undef _FILE_OFFSET_BITS */ + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Type to use in place of in_addr_t when system does not provide it. */ +/* #undef in_addr_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ From 6a28e7ec9ce789b5d1684f6fb92365a1ac244aed Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 12:24:59 +0100 Subject: [PATCH 251/332] Fix #ci_set_analyzer --- .../02967_parallel_replicas_join_algo_and_analyzer.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference index 165504174d0..d7fa419aeab 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference @@ -82,8 +82,8 @@ nested join with analyzer and parallel replicas, both local 420294 420294 420294 -140098 420336 420336 420336 -140112 420378 420378 420378 -140126 -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) +SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both global From f64001ebfab4e7327d754ac139cbd8674b5d0ffe Mon Sep 17 00:00:00 2001 From: Oxide Computer Company Date: Thu, 21 Mar 2024 11:37:26 +0000 Subject: [PATCH 252/332] contrib/curl: Add illumos support --- contrib/curl-cmake/curl_config.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/curl-cmake/curl_config.h b/contrib/curl-cmake/curl_config.h index a38aa60fe6d..4d4c2972f57 100644 --- a/contrib/curl-cmake/curl_config.h +++ b/contrib/curl-cmake/curl_config.h @@ -51,3 +51,8 @@ #define USE_OPENSSL #define USE_THREADS_POSIX #define USE_ARES + +#ifdef __illumos__ +#define HAVE_POSIX_STRERROR_R 1 +#define HAVE_STRERROR_R 1 +#endif From 4700222a0d363a978b12ceb486930679ecd63442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 12:40:07 +0100 Subject: [PATCH 253/332] Fuzzer: Try a different way to wait for the server --- docker/test/fuzzer/run-fuzzer.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 55f4d379005..76661a5b51c 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -181,7 +181,15 @@ function fuzz -- --path db \ --logger.console=0 \ --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 & - server_pid=$(pidof clickhouse-server) + for _ in {1..30} + do + if clickhouse-client --query "select 1" + then + break + fi + sleep 1 + done + server_pid=$(cat /var/run/clickhouse-server/clickhouse-server.pid) kill -0 $server_pid From cb71537c62cf4cb87121bd8d3db0d0602fac7286 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 21 Mar 2024 13:11:56 +0100 Subject: [PATCH 254/332] Ping CI From 7708e0e093a333baf39e9eb3058548cd52eed244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 13:12:59 +0100 Subject: [PATCH 255/332] Disable some tests for SMT --- .../0_stateless/03002_part_log_rmt_fetch_merge_error.sql | 3 ++- .../0_stateless/03002_part_log_rmt_fetch_mutate_error.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql index a7381df5e11..548a8e5570a 100644 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql +++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql @@ -1,4 +1,5 @@ --- Tags: no-replicated-database, no-parallel +-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree +-- SMT: The merge process is completely different from RMT drop table if exists rmt_master; drop table if exists rmt_slave; diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql index f19cb9c73c9..34ba034f798 100644 --- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql +++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql @@ -1,4 +1,5 @@ --- Tags: no-replicated-database, no-parallel +-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree +-- SMT: The merge process is completely different from RMT drop table if exists rmt_master; drop table if exists rmt_slave; From 638754f988cb70511801ec904af85dd4b763d92a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 21 Mar 2024 13:34:32 +0100 Subject: [PATCH 256/332] Fix signal handler for sanitizer signals --- src/Daemon/BaseDaemon.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 289a41bb75e..7fc210a691a 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -275,7 +275,10 @@ public: } readPODBinary(stack_trace, in); - readVectorBinary(thread_frame_pointers, in); + + if (sig != SanitizerTrap) + readVectorBinary(thread_frame_pointers, in); + readBinary(thread_num, in); readPODBinary(thread_ptr, in); @@ -542,6 +545,16 @@ private: #if defined(SANITIZER) + +template +struct ValueHolder +{ + ValueHolder(T value_) : value(value_) + {} + + T value; +}; + extern "C" void __sanitizer_set_death_callback(void (*)()); /// Sanitizers may not expect some function calls from death callback. @@ -559,10 +572,13 @@ static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback() const StackTrace stack_trace; - int sig = SignalListener::SanitizerTrap; - writeBinary(sig, out); + writeBinary(SignalListener::SanitizerTrap, out); writePODBinary(stack_trace, out); - writeBinary(UInt32(getThreadId()), out); + /// We create a dummy struct with a constructor so DISABLE_SANITIZER_INSTRUMENTATION is not applied to it + /// otherwise, Memory sanitizer can't know that values initiialized inside this function are actually initialized + /// because instrumentations are disabled leading to false positives later on + ValueHolder thread_id{static_cast(getThreadId())}; + writeBinary(thread_id.value, out); writePODBinary(current_thread, out); out.next(); From 77e947c44f343d3aab32340c4fda4545fb794a57 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 21 Mar 2024 14:09:01 +0100 Subject: [PATCH 257/332] Avoid `IsADirectoryError: Is a directory contrib/azure` --- tests/ci/style_check.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 797c0ef12d0..2604793c900 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -100,20 +100,28 @@ def is_python(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-script.python" - ) + try: + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-script.python" + ) + except IsADirectoryError: + # Process submodules w/o errors + return False def is_shell(file: Union[Path, str]) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-shellscript" - ) + try: + return bool( + magic.from_file(os.path.join(REPO_COPY, file), mime=True) + == "text/x-shellscript" + ) + except IsADirectoryError: + # Process submodules w/o errors + return False def main(): From e828acd0e5f98d0f6c61c260be9cb428ab0695d6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 Mar 2024 13:17:15 +0000 Subject: [PATCH 258/332] Review fixes. --- src/Storages/System/StorageSystemDetachedParts.cpp | 6 +----- src/Storages/System/StorageSystemPartsBase.cpp | 10 ++-------- src/Storages/VirtualColumnUtils.cpp | 6 +++--- src/Storages/VirtualColumnUtils.h | 3 +++ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 7207e981561..ebcd8d63a52 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -329,7 +329,6 @@ protected: void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes) { - filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes); if (filter_actions_dag) { @@ -344,10 +343,7 @@ void ReadFromSystemDetachedParts::applyFilters(ActionDAGNodes added_filter_nodes filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter, context); } } diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index c81c2c18b2e..11cd6989f31 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -264,10 +264,7 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) filter_by_database = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_database) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter_by_database, context); block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); @@ -276,10 +273,7 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) filter_by_other_columns = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_other_columns) - { - auto empty_block = block.cloneWithColumns(block.cloneEmptyColumns()); - VirtualColumnUtils::filterBlockWithDAG(filter_by_database, empty_block, context); - } + VirtualColumnUtils::buildSetsForDAG(filter_by_other_columns, context); } } diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index c3ac27903c9..e3cbff5f01b 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -53,9 +53,9 @@ namespace DB namespace VirtualColumnUtils { -static void makeSets(const ExpressionActionsPtr & actions, const ContextPtr & context) +void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context) { - for (const auto & node : actions->getNodes()) + for (const auto & node : dag->getNodes()) { if (node.type == ActionsDAG::ActionType::COLUMN) { @@ -78,8 +78,8 @@ static void makeSets(const ExpressionActionsPtr & actions, const ContextPtr & co void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context) { + buildSetsForDAG(dag, context); auto actions = std::make_shared(dag); - makeSets(actions, context); Block block_with_filter = block; actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true); diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h index 83494872cac..62f2e4855b5 100644 --- a/src/Storages/VirtualColumnUtils.h +++ b/src/Storages/VirtualColumnUtils.h @@ -25,6 +25,9 @@ void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, /// Just filters block. Block should contain all the required columns. void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context); +/// Builds sets used by ActionsDAG inplace. +void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context); + /// Recursively checks if all functions used in DAG are deterministic in scope of query. bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node); From 15a61a998a176314debe152b5d59be4ff3f5797a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 21 Mar 2024 14:34:31 +0100 Subject: [PATCH 259/332] Fix logic for run_cpp_check, it had a bug --- tests/ci/style_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 2604793c900..d7f6fa998e9 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -143,8 +143,8 @@ def main(): run_python_check = True if CI and pr_info.number > 0: pr_info.fetch_changed_files() - run_cpp_check = not any( - is_python(file) or is_shell(file) for file in pr_info.changed_files + run_cpp_check = any( + not (is_python(file) or is_shell(file)) for file in pr_info.changed_files ) run_shell_check = any(is_shell(file) for file in pr_info.changed_files) run_python_check = any(is_python(file) for file in pr_info.changed_files) From 48cb228c9a88b1f77c9ec10ce04107edec3e190c Mon Sep 17 00:00:00 2001 From: Shuai li Date: Thu, 21 Mar 2024 21:47:29 +0800 Subject: [PATCH 260/332] Fix addDays cause an error when used datetime64 (#61561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix addDays cause an error when used datetime64 * add exception describe * Update tests/queries/0_stateless/03013_addDays_with_timezone.sql * remove file be executable * fix timezone * fix ci --------- Co-authored-by: János Benjamin Antal --- .../FunctionDateOrDateTimeAddInterval.h | 4 +-- ...21_datetime64_compatibility_long.reference | 32 +++++++++---------- .../03013_addDays_with_timezone.reference | 1 + .../03013_addDays_with_timezone.sql | 1 + 4 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/03013_addDays_with_timezone.reference create mode 100644 tests/queries/0_stateless/03013_addDays_with_timezone.sql diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index b8c0d27c42e..f50b1415622 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -621,9 +621,9 @@ public: } else { - if (!WhichDataType(arguments[0].type).isDateTime()) + if (!WhichDataType(arguments[0].type).isDateTimeOrDateTime64()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument of function {}. " - "Must be a DateTime", arguments[0].type->getName(), getName()); + "Must be a DateTime/DateTime64", arguments[0].type->getName(), getName()); if (!WhichDataType(arguments[2].type).isString()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of third argument of function {}. " diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference index a946a114bf4..74b7b207661 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference @@ -241,82 +241,82 @@ SELECT toYYYYMMDDhhmmss(N, \'Asia/Istanbul\') SELECT addYears(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2020-09-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2020-09-16 19:20:11.234" ------------------------------------------ SELECT addMonths(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-10-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-10-16 19:20:11.234" ------------------------------------------ SELECT addWeeks(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-23 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-23 19:20:11.234" ------------------------------------------ SELECT addDays(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-17 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-17 19:20:11.234" ------------------------------------------ SELECT addHours(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 20:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 20:20:11.234" ------------------------------------------ SELECT addMinutes(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:21:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:21:11.234" ------------------------------------------ SELECT addSeconds(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:20:12" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT addQuarters(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-12-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-12-16 19:20:11.234" ------------------------------------------ SELECT subtractYears(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2018-09-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2018-09-16 19:20:11.234" ------------------------------------------ SELECT subtractMonths(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-08-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-08-16 19:20:11.234" ------------------------------------------ SELECT subtractWeeks(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-09 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-09 19:20:11.234" ------------------------------------------ SELECT subtractDays(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-15 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-15 19:20:11.234" ------------------------------------------ SELECT subtractHours(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 18:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 18:20:11.234" ------------------------------------------ SELECT subtractMinutes(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:19:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:19:11.234" ------------------------------------------ SELECT subtractSeconds(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-09-16 19:20:10" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT subtractQuarters(N, 1, \'Asia/Istanbul\') Code: 43 "DateTime('Asia/Istanbul')","2019-06-16 19:20:11" -Code: 43 +"DateTime64(3, 'Asia/Istanbul')","2019-06-16 19:20:11.234" ------------------------------------------ SELECT CAST(N as DateTime(\'Europe/Minsk\')) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" diff --git a/tests/queries/0_stateless/03013_addDays_with_timezone.reference b/tests/queries/0_stateless/03013_addDays_with_timezone.reference new file mode 100644 index 00000000000..6a8fa4f8a2c --- /dev/null +++ b/tests/queries/0_stateless/03013_addDays_with_timezone.reference @@ -0,0 +1 @@ +2024-01-11 00:00:00.000000 diff --git a/tests/queries/0_stateless/03013_addDays_with_timezone.sql b/tests/queries/0_stateless/03013_addDays_with_timezone.sql new file mode 100644 index 00000000000..eb822d53898 --- /dev/null +++ b/tests/queries/0_stateless/03013_addDays_with_timezone.sql @@ -0,0 +1 @@ +select addDays(toDateTime64('2024-01-01', 6, 'Asia/Shanghai'), 10, 'Asia/Shanghai'); From 7c173e4dbc9606f2d0f4de6d61bbfe214997891c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 16:03:50 +0100 Subject: [PATCH 261/332] Analyzer: fix group_by_use_nulls #ci_set_analyzer --- src/Planner/PlannerExpressionAnalysis.cpp | 8 ++++++-- .../03014_analyzer_group_by_use_nulls.reference | 2 ++ .../0_stateless/03014_analyzer_group_by_use_nulls.sql | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference create mode 100644 tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index c664317b250..441e2ac4b9e 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -111,7 +113,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name); + auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -161,7 +164,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - available_columns_after_aggregation.emplace_back(expression_dag_node->column, expression_type_after_aggregation, expression_dag_node->result_name); + auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); diff --git a/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference new file mode 100644 index 00000000000..b50fdcee209 --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.reference @@ -0,0 +1,2 @@ +a b +a b diff --git a/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql new file mode 100644 index 00000000000..a1c302465de --- /dev/null +++ b/tests/queries/0_stateless/03014_analyzer_group_by_use_nulls.sql @@ -0,0 +1 @@ +SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS group_by_use_nulls = 1; From f36e19917fcb5d8acc290bee77ff1ffd9ceca587 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 21 Mar 2024 15:27:39 +0000 Subject: [PATCH 262/332] Disable analyzer for EXPLAIN SYNTAX queries --- src/Interpreters/InterpreterFactory.cpp | 4 ++++ ...allel_replicas_cte_explain_syntax_crash.sql | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 0b87ec3cc0e..387d056ffe7 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -219,6 +219,10 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte } else if (query->as()) { + const auto kind = query->as()->getKind(); + if (kind == ASTExplainQuery::ParsedAST || kind == ASTExplainQuery::AnalyzedSyntax) + context->setSetting("allow_experimental_analyzer", false); + interpreter_name = "InterpreterExplainQuery"; } else if (query->as()) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql index f46817d5e82..df4ec9d26a3 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -3,29 +3,26 @@ DROP TABLE IF EXISTS numbers_1e6__fuzz_33; CREATE TABLE numbers_1e6__fuzz_34 ( - `n` LowCardinality(Nullable(UInt8)) + n UInt64 ) ENGINE = MergeTree ORDER BY n -SETTINGS allow_nullable_key = 1 AS SELECT * -FROM numbers(1000000) -SETTINGS allow_suspicious_low_cardinality_types = 1; +FROM numbers(10); CREATE TABLE numbers_1e6__fuzz_33 ( - `n` LowCardinality(Nullable(UInt8)) + n UInt64 ) ENGINE = MergeTree ORDER BY n -SETTINGS allow_nullable_key = 1 AS SELECT * -FROM numbers(1000000) -SETTINGS allow_suspicious_low_cardinality_types = 1; +FROM numbers(10); SET allow_experimental_analyzer = 1; -SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0; + EXPLAIN SYNTAX WITH cte1 AS @@ -41,3 +38,6 @@ WITH ) SELECT count() FROM cte2; + +DROP TABLE numbers_1e6__fuzz_34; +DROP TABLE numbers_1e6__fuzz_33; From 37b34d6ed159b76528e5bc8b1d74705eae0ba97d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 16:28:45 +0100 Subject: [PATCH 263/332] Analyzer: Clear list of broken integration tests --- tests/analyzer_integration_broken_tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt index 56fde20c363..e69de29bb2d 100644 --- a/tests/analyzer_integration_broken_tests.txt +++ b/tests/analyzer_integration_broken_tests.txt @@ -1 +0,0 @@ -test_concurrent_backups_s3/test.py::test_concurrent_backups From 29a52419972ca8bac81d56ced30d871495091cdb Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 20 Mar 2024 17:18:08 +0100 Subject: [PATCH 264/332] fix build --- src/Common/Config/AbstractConfigurationComparison.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index e241a540cc1..607b583cf31 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -41,7 +41,7 @@ namespace #if defined(ABORT_ON_LOGICAL_ERROR) /// Compound `ignore_keys` are not yet implemented. for (const auto & ignore_key : *ignore_keys) - chassert(ignore_key.find(".") == std::string_view::npos); + chassert(ignore_key.find('.') == std::string_view::npos); #endif } From 63631ee58721729bd82d45a854074546c38be5f8 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 21 Mar 2024 15:35:05 +0000 Subject: [PATCH 265/332] Remove debug code --- src/Storages/StorageMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 19f951c8dc4..c87681a1418 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -219,7 +219,6 @@ void StorageMergeTree::read( Block header; if (local_context->getSettingsRef().allow_experimental_analyzer) { - chassert(query_info.query_tree); QueryTreeNodePtr modified_query_tree = query_info.query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, local_context); modified_query_tree = buildQueryTreeForShard(query_info.planner_context, modified_query_tree); From 110b08f666c158d0f0cabbeff975e56b57f2286f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 18:38:57 +0100 Subject: [PATCH 266/332] Fixup #ci_set_analyzer --- src/Planner/PlannerExpressionAnalysis.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index 441e2ac4b9e..dd3769ee10b 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -113,7 +113,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); @@ -164,7 +164,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP continue; auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; - auto column_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); From b275e06aca70a334babf74412fea6cf728664bca Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 21 Mar 2024 19:02:46 +0100 Subject: [PATCH 267/332] Add test for 57820 #ci_set_analyzer --- .../queries/0_stateless/03018_analyzer_greater_null.reference | 1 + tests/queries/0_stateless/03018_analyzer_greater_null.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/03018_analyzer_greater_null.reference create mode 100644 tests/queries/0_stateless/03018_analyzer_greater_null.sql diff --git a/tests/queries/0_stateless/03018_analyzer_greater_null.reference b/tests/queries/0_stateless/03018_analyzer_greater_null.reference new file mode 100644 index 00000000000..1e254619239 --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_greater_null.reference @@ -0,0 +1 @@ +\N 0 diff --git a/tests/queries/0_stateless/03018_analyzer_greater_null.sql b/tests/queries/0_stateless/03018_analyzer_greater_null.sql new file mode 100644 index 00000000000..66ea53e451f --- /dev/null +++ b/tests/queries/0_stateless/03018_analyzer_greater_null.sql @@ -0,0 +1,4 @@ +SELECT + max(NULL > 255) > NULL AS a, + count(NULL > 1.) > 1048577 +FROM numbers(10); From 958659584957ff419a9305d9c7edee5703fedbdc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Mar 2024 19:09:40 +0100 Subject: [PATCH 268/332] Fix clang-tidy --- src/Functions/FunctionsTimeWindow.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 65cc7e9e87c..d52b76bec91 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -119,7 +119,7 @@ template<> \ struct AddTime \ { \ static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) \ - { return static_cast(t + delta * INTERVAL); } \ + { return static_cast(t + delta * (INTERVAL)); } \ }; ADD_TIME(Day, 86400) ADD_TIME(Hour, 3600) From 89d80a4d65be42986f8d710c6f2ac5305491b80b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 21 Mar 2024 18:16:31 +0000 Subject: [PATCH 269/332] Redo fix. --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 57 ++++++++++++++++++----- src/Functions/randConstant.cpp | 16 ++----- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 2a46417ee98..834bc2e5813 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -799,7 +799,13 @@ struct IdentifierResolveScope /// Node hash to mask id map std::shared_ptr> projection_mask_map; - std::map rand_constant_cache; + struct ResolvedFunctionsCache + { + FunctionOverloadResolverPtr resolver; + std::map cache; + }; + + std::map functions_cache; [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const { @@ -925,6 +931,24 @@ struct IdentifierResolveScope } }; +IQueryTreeNode::Hash getHashForFunctionArguments(const ColumnsWithTypeAndName & arguments) +{ + SipHash hash; + for (const auto & arg : arguments) + { + auto type_name = arg.type->getName(); + hash.update(type_name.c_str(), type_name.size()); + + if (arg.column) + { + if (const auto * col_const = typeid_cast(arg.column.get())) + col_const->updateHashWithValue(0, hash); + } + } + + return getSipHash128AsPair(hash); +} + /** Visitor that extracts expression and function aliases from node and initialize scope tables with it. * Does not go into child lambdas and queries. @@ -5534,21 +5558,19 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters); bool is_executable_udf = true; + IdentifierResolveScope::ResolvedFunctionsCache * function_cache = nullptr; + if (!function) { /// This is a hack to allow a query like `select randConstant(), randConstant(), randConstant()`. /// Function randConstant() would return the same value for the same arguments (in scope). - if (function_name == "randConstant") - { - auto hash = function_node_ptr->getTreeHash(); - auto & func = scope.rand_constant_cache[hash]; - if (!func) - func = FunctionFactory::instance().tryGet(function_name, scope.context); - function = func; - } - else - function = FunctionFactory::instance().tryGet(function_name, scope.context); + auto hash = function_node_ptr->getTreeHash(); + function_cache = &scope.functions_cache[hash]; + if (!function_cache->resolver) + function_cache->resolver = FunctionFactory::instance().tryGet(function_name, scope.context); + + function = function_cache->resolver; is_executable_udf = false; } @@ -5773,7 +5795,18 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi try { - auto function_base = function->build(argument_columns); + FunctionBasePtr function_base; + if (function_cache) + { + auto args_hash = getHashForFunctionArguments(argument_columns); + auto & cached_function = function_cache->cache[args_hash]; + if (!cached_function) + cached_function = function->build(argument_columns); + + function_base = cached_function; + } + else + function_base = function->build(argument_columns); /// Do not constant fold get scalar functions bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" || diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index 5872ab8bb34..22ce6d88ea6 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -88,14 +88,6 @@ public: return std::make_unique>(); } - RandomConstantOverloadResolver() - { - typename ColumnVector::Container vec_to(1); - - TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); - value = vec_to[0]; - } - DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override { size_t number_of_arguments = data_types.size(); @@ -113,11 +105,13 @@ public: if (!arguments.empty()) argument_types.emplace_back(arguments.back().type); + typename ColumnVector::Container vec_to(1); + + TargetSpecific::Default::RandImpl::execute(reinterpret_cast(vec_to.data()), sizeof(ToType)); + ToType value = vec_to[0]; + return std::make_unique>(value, argument_types, return_type); } - -private: - ToType value; }; struct NameRandConstant { static constexpr auto name = "randConstant"; }; From d6b515b6bf6069b980509bfa0bd4f8fdcd088acf Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 17:20:08 +0000 Subject: [PATCH 270/332] CI: modify CI from PR body #do_not_test #job_style_check --- .github/PULL_REQUEST_TEMPLATE.md | 30 ++++++++++++++++++++++++++++++ .gitmessage | 2 +- tests/ci/ci.py | 21 +++++++++++++++------ tests/ci/pr_info.py | 3 +++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7fb2abebbbb..4db0737c959 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,3 +40,33 @@ At a minimum, the following information should be added (but add more as needed) > Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/ + +--- +### Modify your CI run: +##### NOTE: +- if your merge the PR with modified CI you **MUST** know what you are doing. +- modifiers can be applied only if set before CI starts +- remove `!` to apply +- return all `!` to restore defaults +``` +!#ci_set_ - to run only preconfigured set of tests, e.g.: +!#ci_set_arm - to run only integration tests on ARM +!#ci_set_integration - to run only integration tests on AMD +!#ci_set_analyzer - to run only tests for analyzer +NOTE: you can configure your own ci set +``` +``` +!#job_ - to run only specified job, e.g.: +!#job_stateless_tests_release +!#job_package_debug +!#job_style_check +!#job_integration_tests_asan +``` +``` +!#batch_2 - to run only 2nd batch for all multi-batch jobs +!#btach_1_2_3 - to run only 1, 2, 3rd batch for all multi-batch jobs +``` +``` +!#no_merge_commit - to disable merge commit (no merge from master) +!#do_not_test - to disable whole CI (except style check) +``` diff --git a/.gitmessage b/.gitmessage index 2ad30596de6..797446edd49 100644 --- a/.gitmessage +++ b/.gitmessage @@ -26,4 +26,4 @@ ## To run only specified batches for multi-batch job(s) #batch_2 -#btach_1_2_3 +#batch_1_2_3 diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 29906e6571f..cd63514cb6a 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1407,15 +1407,25 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: print("... CI report update - done") -def _fetch_commit_tokens(message: str) -> List[str]: - pattern = r"#[\w-]+" - matches = [match[1:] for match in re.findall(pattern, message)] +def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]: + pattern = r"([^!]|^)#(\w+)" + matches = [match[-1] for match in re.findall(pattern, message)] res = [ match for match in matches if match in Labels or match.startswith("job_") or match.startswith("batch_") ] - return res + print(f"CI modifyers from commit message: [{res}]") + res_2 = [] + if pr_info.is_pr(): + matches = [match[-1] for match in re.findall(pattern, pr_info.body)] + res_2 = [ + match + for match in matches + if match in Labels or match.startswith("job_") or match.startswith("batch_") + ] + print(f"CI modifyers from PR body: [{res_2}]") + return list(set(res + res_2)) def _upload_build_artifacts( @@ -1701,8 +1711,7 @@ def main() -> int: message = args.commit_message or git_runner.run( f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" ) - tokens = _fetch_commit_tokens(message) - print(f"Commit message tokens: [{tokens}]") + tokens = _fetch_commit_tokens(message, pr_info) if Labels.NO_MERGE_COMMIT in tokens and CI: git_runner.run(f"{GIT_PREFIX} checkout {pr_info.sha}") git_ref = git_runner.run(f"{GIT_PREFIX} rev-parse HEAD") diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 9bd30f3c58e..84f2db4002d 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -316,6 +316,9 @@ class PRInfo: def is_release_branch(self) -> bool: return self.number == 0 + def is_pr(self): + return self.event_type == EventType.PULL_REQUEST + def is_scheduled(self): return self.event_type == EventType.SCHEDULE From 54c5ea8d6d1e13b8151bbfcc7cffd8f1de5c616b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 21 Mar 2024 21:14:24 +0100 Subject: [PATCH 271/332] FunctionsConversion: Remove some templates --- src/Functions/FunctionsConversion.cpp | 164 ++++++++++++-------------- 1 file changed, 78 insertions(+), 86 deletions(-) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index ceff4f3fd7e..9262389e593 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1127,23 +1127,24 @@ struct AccurateOrNullConvertStrategyAdditions UInt32 scale { 0 }; }; - -struct ConvertDefaultBehaviorTag {}; -struct ConvertReturnNullOnErrorTag {}; -struct ConvertReturnZeroOnErrorTag {}; +enum class BehaviourOnErrorFromString +{ + ConvertDefaultBehaviorTag, + ConvertReturnNullOnErrorTag, + ConvertReturnZeroOnErrorTag +}; /** Conversion of number types to each other, enums to numbers, dates and datetimes to numbers and back: done by straight assignment. * (Date is represented internally as number of days from some day; DateTime - as unix timestamp) */ template struct ConvertImpl { template static ColumnPtr NO_SANITIZE_UNDEFINED execute( const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type [[maybe_unused]], size_t input_rows_count, - Additions additions = Additions()) + BehaviourOnErrorFromString from_string_tag [[maybe_unused]], Additions additions = Additions()) { const ColumnWithTypeAndName & named_from = arguments[0]; @@ -1176,8 +1177,7 @@ struct ConvertImpl return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); } - else if constexpr (std::is_same_v && std::is_same_v - && std::is_same_v) + else if constexpr (std::is_same_v && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1194,8 +1194,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1203,8 +1202,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1214,8 +1212,7 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1223,8 +1220,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1232,8 +1228,7 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1243,8 +1238,7 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1254,15 +1248,13 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1271,8 +1263,7 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count); @@ -1282,15 +1273,13 @@ struct ConvertImpl || std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1298,23 +1287,20 @@ struct ConvertImpl else if constexpr (( std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl, false>::template execute( arguments, result_type, input_rows_count, additions); } /// Conversion of DateTime64 to Date or DateTime: discards fractional part. else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl>, false>::template execute( arguments, result_type, input_rows_count, additions); } else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl>, false>::template execute( arguments, result_type, input_rows_count, additions); @@ -1324,8 +1310,7 @@ struct ConvertImpl std::is_same_v || std::is_same_v || std::is_same_v) - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { return DateTimeTransformImpl::template execute( arguments, result_type, input_rows_count, additions); @@ -1443,8 +1428,7 @@ struct ConvertImpl /// Conversion from FixedString to String. /// Cutting sequences of zero bytes from end of strings. else if constexpr (std::is_same_v - && std::is_same_v - && std::is_same_v) + && std::is_same_v) { ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column); const auto & nested = columnGetNested(arguments[0]); @@ -1488,8 +1472,7 @@ struct ConvertImpl throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", arguments[0].column->getName(), Name::name); } - else if constexpr (std::is_same_v - && std::is_same_v) + else if constexpr (std::is_same_v) { /// Anything else to String. @@ -1550,27 +1533,35 @@ struct ConvertImpl && std::is_same_v && std::is_same_v) { - return ConvertImpl::template execute( - arguments, result_type, input_rows_count); + return ConvertImpl::template execute( + arguments, result_type, input_rows_count, from_string_tag); } - else if constexpr ((std::is_same_v || std::is_same_v) - && std::is_same_v) + else if constexpr ((std::is_same_v || std::is_same_v)) { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); - } - else if constexpr ((std::is_same_v || std::is_same_v) - && std::is_same_v) - { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); - } - else if constexpr ((std::is_same_v || std::is_same_v) - && is_any_of - && std::is_same_v) - { - return ConvertThroughParsing::execute( - arguments, result_type, input_rows_count, additions); + switch (from_string_tag) + { + case BehaviourOnErrorFromString::ConvertDefaultBehaviorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + case BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + case BehaviourOnErrorFromString::ConvertReturnZeroOnErrorTag: + return ConvertThroughParsing::execute( + arguments, result_type, input_rows_count, additions); + } } else { @@ -2165,12 +2156,11 @@ private: if (context) date_time_overflow_behavior = context->getSettingsRef().date_time_overflow_behavior.value; - auto call = [&](const auto & types, const auto & tag) -> bool + auto call = [&](const auto & types, BehaviourOnErrorFromString from_string_tag) -> bool { using Types = std::decay_t; using LeftDataType = typename Types::LeftType; using RightDataType = typename Types::RightType; - using SpecialTag = std::decay_t; if constexpr (IsDataTypeDecimal) { @@ -2191,13 +2181,13 @@ private: switch (date_time_overflow_behavior) { case FormatSettings::DateTimeOverflowBehavior::Throw: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; case FormatSettings::DateTimeOverflowBehavior::Ignore: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; case FormatSettings::DateTimeOverflowBehavior::Saturate: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, scale); break; } @@ -2208,20 +2198,20 @@ private: switch (date_time_overflow_behavior) { case FormatSettings::DateTimeOverflowBehavior::Throw: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; case FormatSettings::DateTimeOverflowBehavior::Ignore: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; case FormatSettings::DateTimeOverflowBehavior::Saturate: - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, dt64->getScale()); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag, dt64->getScale()); break; } } #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::execute( \ - arguments, result_type, input_rows_count); \ + result_column = ConvertImpl::execute( \ + arguments, result_type, input_rows_count, from_string_tag); \ break; else if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) @@ -2262,7 +2252,7 @@ private: } #undef GENERATE_OVERFLOW_MODE_CASE else - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, from_string_tag); return true; }; @@ -2275,7 +2265,7 @@ private: if (to_datetime64 || scale != 0) /// When scale = 0, the data type is DateTime otherwise the data type is DateTime64 { - if (!callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{})) + if (!callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0].type->getName(), getName()); @@ -2292,23 +2282,25 @@ private: bool done = false; if constexpr (is_any_of) { - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); } else { bool cast_ipv4_ipv6_default_on_conversion_error = false; if constexpr (is_any_of) + { if (context && (cast_ipv4_ipv6_default_on_conversion_error = context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error)) - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertReturnZeroOnErrorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertReturnZeroOnErrorTag); + } if (!cast_ipv4_ipv6_default_on_conversion_error) { /// We should use ConvertFromStringExceptionMode::Null mode when converting from String (or FixedString) /// to Nullable type, to avoid 'value is too short' error on attempt to parse empty string from NULL values. if (to_nullable && WhichDataType(from_type).isStringOrFixedString()) - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertReturnNullOnErrorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag); else - done = callOnIndexAndDataType(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{}); + done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); } } @@ -3185,8 +3177,8 @@ private: { #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE, ADDITIONS) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::execute( \ - arguments, result_type, input_rows_count, ADDITIONS()); \ + result_column = ConvertImpl::execute( \ + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, ADDITIONS()); \ break; if (wrapper_cast_type == CastType::accurate) { @@ -3215,8 +3207,8 @@ private: { #define GENERATE_OVERFLOW_MODE_CASE(OVERFLOW_MODE, ADDITIONS) \ case FormatSettings::DateTimeOverflowBehavior::OVERFLOW_MODE: \ - result_column = ConvertImpl::template execute( \ -arguments, result_type, input_rows_count); \ + result_column = ConvertImpl::template execute( \ +arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag); \ break; if (wrapper_cast_type == CastType::accurate) { @@ -3376,7 +3368,7 @@ arguments, result_type, input_rows_count); \ AccurateConvertStrategyAdditions additions; additions.scale = scale; result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, additions); + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, additions); return true; } @@ -3385,7 +3377,7 @@ arguments, result_type, input_rows_count); \ AccurateOrNullConvertStrategyAdditions additions; additions.scale = scale; result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, additions); + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, additions); return true; } @@ -3397,14 +3389,14 @@ arguments, result_type, input_rows_count); \ /// Consistent with CAST(Nullable(String) AS Nullable(Numbers)) /// In case when converting to Nullable type, we apply different parsing rule, /// that will not throw an exception but return NULL in case of malformed input. - result_column = ConvertImpl::execute( - arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute( + arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertReturnNullOnErrorTag, scale); return true; } } - result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, scale); + result_column = ConvertImpl::execute(arguments, result_type, input_rows_count, BehaviourOnErrorFromString::ConvertDefaultBehaviorTag, scale); return true; }); From 389566f895979104f4931ddb5a2d36be43e1eecb Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 20:23:06 +0000 Subject: [PATCH 272/332] quick fix for style check --- tests/ci/style_check.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index d7f6fa998e9..05788aad5ea 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -8,9 +8,8 @@ import subprocess import sys from concurrent.futures import ProcessPoolExecutor from pathlib import Path -from typing import List, Tuple, Union +from typing import List, Tuple -import magic from docker_images_helper import get_docker_image, pull_image from env_helper import CI, REPO_COPY, TEMP_PATH from git_helper import GIT_PREFIX, git_runner @@ -96,32 +95,34 @@ def commit_push_staged(pr_info: PRInfo) -> None: git_runner(push_cmd) -def is_python(file: Union[Path, str]) -> bool: +def is_python(file: str) -> bool: """returns if the changed file in the repository is python script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-script.python" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-script.python" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".py") -def is_shell(file: Union[Path, str]) -> bool: +def is_shell(file: str) -> bool: """returns if the changed file in the repository is shell script""" # WARNING: python-magic v2:0.4.24-2 is used in ubuntu 22.04, # and `Support os.PathLike values in magic.from_file` is only from 0.4.25 - try: - return bool( - magic.from_file(os.path.join(REPO_COPY, file), mime=True) - == "text/x-shellscript" - ) - except IsADirectoryError: - # Process submodules w/o errors - return False + # try: + # return bool( + # magic.from_file(os.path.join(REPO_COPY, file), mime=True) + # == "text/x-shellscript" + # ) + # except IsADirectoryError: + # # Process submodules w/o errors + # return False + return file.endswith(".sh") def main(): From b91d478c937fe73a7f92585c3b8d30b8732d5975 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 Mar 2024 00:56:57 +0100 Subject: [PATCH 273/332] There is no such thing as broken tests --- tests/analyzer_integration_broken_tests.txt | 0 tests/integration/ci-runner.py | 32 ++++----------------- 2 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 tests/analyzer_integration_broken_tests.txt diff --git a/tests/analyzer_integration_broken_tests.txt b/tests/analyzer_integration_broken_tests.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e7f691d2237..32a60c8fa47 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -475,7 +475,7 @@ class ClickhouseIntegrationTestsRunner: return result @staticmethod - def _update_counters(main_counters, current_counters, broken_tests): + def _update_counters(main_counters, current_counters): for test in current_counters["PASSED"]: if test not in main_counters["PASSED"]: if test in main_counters["FAILED"]: @@ -483,21 +483,14 @@ class ClickhouseIntegrationTestsRunner: if test in main_counters["BROKEN"]: main_counters["BROKEN"].remove(test) - if test not in broken_tests: - main_counters["PASSED"].append(test) - else: - main_counters["NOT_FAILED"].append(test) + main_counters["PASSED"].append(test) for state in ("ERROR", "FAILED"): for test in current_counters[state]: if test in main_counters["PASSED"]: main_counters["PASSED"].remove(test) - if test not in broken_tests: - if test not in main_counters[state]: - main_counters[state].append(test) - else: - if test not in main_counters["BROKEN"]: - main_counters["BROKEN"].append(test) + if test not in main_counters[state]: + main_counters[state].append(test) for state in ("SKIPPED",): for test in current_counters[state]: @@ -564,7 +557,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ): try: return self.run_test_group( @@ -573,7 +565,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ) except Exception as e: logging.info("Failed to run %s:\n%s", test_group, e) @@ -596,7 +587,6 @@ class ClickhouseIntegrationTestsRunner: tests_in_group, num_tries, num_workers, - broken_tests, ): counters = { "ERROR": [], @@ -706,7 +696,7 @@ class ClickhouseIntegrationTestsRunner: ) times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) - self._update_counters(counters, new_counters, broken_tests) + self._update_counters(counters, new_counters) for test_name, test_time in new_tests_times.items(): tests_times[test_name] = test_time @@ -915,20 +905,10 @@ class ClickhouseIntegrationTestsRunner: logging.info("Shuffling test groups") random.shuffle(items_to_run) - broken_tests = [] - if self.use_analyzer: - with open( - f"{repo_path}/tests/analyzer_integration_broken_tests.txt", - "r", - encoding="utf-8", - ) as f: - broken_tests = f.read().splitlines() - logging.info("Broken tests in the list: %s", len(broken_tests)) - for group, tests in items_to_run: logging.info("Running test group %s containing %s tests", group, len(tests)) group_counters, group_test_times, log_paths = self.try_run_test_group( - repo_path, group, tests, MAX_RETRY, NUM_WORKERS, broken_tests + repo_path, group, tests, MAX_RETRY, NUM_WORKERS ) total_tests = 0 for counter, value in group_counters.items(): From 7d7025a74edf9c779cad98ca87d32252c5213e31 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 21 Mar 2024 22:20:33 +0000 Subject: [PATCH 274/332] CI: modify it --- .github/PULL_REQUEST_TEMPLATE.md | 53 ++++++++++++++++---------------- tests/ci/ci.py | 9 +++--- tests/ci/ci_config.py | 46 ++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4db0737c959..fede3fe519d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -43,30 +43,29 @@ At a minimum, the following information should be added (but add more as needed) --- ### Modify your CI run: -##### NOTE: -- if your merge the PR with modified CI you **MUST** know what you are doing. -- modifiers can be applied only if set before CI starts -- remove `!` to apply -- return all `!` to restore defaults -``` -!#ci_set_ - to run only preconfigured set of tests, e.g.: -!#ci_set_arm - to run only integration tests on ARM -!#ci_set_integration - to run only integration tests on AMD -!#ci_set_analyzer - to run only tests for analyzer -NOTE: you can configure your own ci set -``` -``` -!#job_ - to run only specified job, e.g.: -!#job_stateless_tests_release -!#job_package_debug -!#job_style_check -!#job_integration_tests_asan -``` -``` -!#batch_2 - to run only 2nd batch for all multi-batch jobs -!#btach_1_2_3 - to run only 1, 2, 3rd batch for all multi-batch jobs -``` -``` -!#no_merge_commit - to disable merge commit (no merge from master) -!#do_not_test - to disable whole CI (except style check) -``` +**NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing +**NOTE:** Set desired options before CI starts or re-push after updates + +#### Run only: +- [ ] Integration tests +- [ ] Integration tests (arm64) +- [ ] Stateless tests (release) +- [ ] Stateless tests (asan) +- [ ] Stateful tests (release) +- [ ] Stateful tests (asan) +- [ ] No sanitizers +- [ ] Tests with analyzer +- [ ] Fast tests +- [ ] Only package_debug build +- [ ] Add your CI variant description here + +#### CI options: +- [ ] do not test (only style check) +- [ ] disable merge-commit (no merge from master before tests) +- [ ] disable CI cache (job reuse) + +#### Only specified batches in multi-batch jobs: +- [ ] 1 +- [ ] 2 +- [ ] 3 +- [ ] 4 diff --git a/tests/ci/ci.py b/tests/ci/ci.py index cd63514cb6a..514189a8b8a 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -1248,9 +1248,7 @@ def _configure_jobs( for token_ in ci_controlling_tokens: label_config = CI_CONFIG.get_label_config(token_) assert label_config, f"Unknonwn token [{token_}]" - print( - f"NOTE: CI controlling token: [{ci_controlling_tokens}], add jobs: [{label_config.run_jobs}]" - ) + print(f"NOTE: CI modifier: [{token_}], add jobs: [{label_config.run_jobs}]") jobs_to_do_requested += label_config.run_jobs # handle specific job requests @@ -1264,7 +1262,7 @@ def _configure_jobs( for job in requested_jobs: job_with_parents = CI_CONFIG.get_job_with_parents(job) print( - f"NOTE: CI controlling token: [#job_{job}], add jobs: [{job_with_parents}]" + f"NOTE: CI modifier: [#job_{job}], add jobs: [{job_with_parents}]" ) # always add requested job itself, even if it could be skipped jobs_to_do_requested.append(job_with_parents[0]) @@ -1273,6 +1271,7 @@ def _configure_jobs( jobs_to_do_requested.append(parent) if jobs_to_do_requested: + jobs_to_do_requested = list(set(jobs_to_do_requested)) print( f"NOTE: Only specific job(s) were requested by commit message tokens: [{jobs_to_do_requested}]" ) @@ -1408,7 +1407,7 @@ def _update_gh_statuses_action(indata: Dict, s3: S3Helper) -> None: def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]: - pattern = r"([^!]|^)#(\w+)" + pattern = r"(#|- \[x\] +